--- //depot/vendor/freebsd/src/sys/kern/kern_descrip.c	2004/02/25 16:30:40
+++ //depot/user/rwatson/netperf/sys/kern/kern_descrip.c	2004/03/03 20:58:29
@@ -2042,6 +2042,8 @@
 fputsock(struct socket *so)
 {
 
+	NET_ASSERT_GIANT();
+	SOCK_LOCK(so);
 	sorele(so);
 }
 
@@ -2067,6 +2069,10 @@
 	}
 	/* We have the last ref so we can proceed without the file lock. */
 	FILE_UNLOCK(fp);
+	/*
+	 * XXXRW: With sockets locked, can we push this Giant grab into
+	 * the purely VFS case?
+	 */
 	mtx_lock(&Giant);
 	if (fp->f_count < 0)
 		panic("fdrop: count < 0");
--- //depot/vendor/freebsd/src/sys/kern/kern_timeout.c	2003/12/07 12:05:33
+++ //depot/user/rwatson/netperf/sys/kern/kern_timeout.c	2004/03/08 08:37:54
@@ -47,6 +47,7 @@
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/sbuf.h>
 #include <sys/sysctl.h>
 
 static int avg_depth;
@@ -58,6 +59,89 @@
 static int avg_mpcalls;
 SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0,
     "Average number of MP callouts made per softclock call. Units = 1/1000");
+
+/*-
+ * Sampling buffer of function pointers executed by timeouts and callouts.
+ * This circular buffer wraps when it fills, and uses an inefficient
+ * sbuf-based sysctl to dump sample data to userspace.  Sysctls can select
+ * to monitor mpsafe and !mpsafe callouts/timeouts as desired.  Suggested
+ * use is: (1) set sample of interest (mpsafe/notmpsafe), (2) reset the
+ * buffer, (3) do some benchmark/test, (5) disable sampling, (6) dump
+ * buffer.
+ *
+ * XXX: ifdef TIMEOUT_SAMPLING?
+ */
+
+#define	MAXFUNC	200000
+static void * func_array[MAXFUNC];
+static int array_off;
+
+static void
+push_cfunc(void *ptr)
+{
+
+	/* XXX */
+	func_array[array_off % MAXFUNC] = ptr;
+	array_off++;
+}
+
+static int
+sysctl_cfunc(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf sb;
+	int error, i;
+
+	if (req->newptr != NULL)
+		return (EINVAL);
+
+	sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND);
+
+	for (i = 0; i < MAXFUNC; i++) {
+		if (func_array[i] == NULL)
+			break;
+		sbuf_printf(&sb, "%p ", func_array[i]);
+	}
+	sbuf_finish(&sb);
+
+	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb) + 1, req);
+
+	sbuf_delete(&sb);
+
+	return (error);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, to_cfunc, CTLTYPE_STRING|CTLFLAG_RD, 0, 0,
+    sysctl_cfunc, "A", "callout/timeout sample");
+
+static int
+sysctl_cfunc_reset(SYSCTL_HANDLER_ARGS)
+{
+	int dummy, error;
+
+	dummy = 0;
+	error = sysctl_handle_int(oidp, &dummy, 0, req);
+	if (error)
+		return (error);
+
+	if (dummy != 0) {
+		bzero(func_array, sizeof(void *) * MAXFUNC);
+		array_off = 0;
+	}
+
+	return (0);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, to_cfunc_reset, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
+    sysctl_cfunc_reset, "I", "Reset sample");
+
+static int cfunc_sample_mpsafe;
+static int cfunc_sample_notmpsafe;
+
+SYSCTL_INT(_debug, OID_AUTO, to_cfunc_mpsafe, CTLFLAG_RW,
+    &cfunc_sample_mpsafe, 0, "Sample mpsafe callouts");
+SYSCTL_INT(_debug, OID_AUTO, to_cfunc_notmpsafe, CTLFLAG_RW,
+    &cfunc_sample_notmpsafe, 0, "Sample !mpsafe callouts");
+
 /*
  * TODO:
  *	allocate more timeout table slots when table overflows.
@@ -216,8 +300,12 @@
 				if (!(c_flags & CALLOUT_MPSAFE)) {
 					mtx_lock(&Giant);
 					gcalls++;
+					if (cfunc_sample_mpsafe)
+						push_cfunc(c_func);
 				} else {
 					mpcalls++;
+					if (cfunc_sample_notmpsafe)
+						push_cfunc(c_func);
 				}
 #ifdef DIAGNOSTIC
 				binuptime(&bt1);
--- //depot/vendor/freebsd/src/sys/kern/subr_log.c	2004/02/21 13:16:09
+++ //depot/user/rwatson/netperf/sys/kern/subr_log.c	2004/03/06 09:01:33
@@ -87,7 +87,12 @@
 	struct	callout sc_callout;	/* callout to wakeup syslog  */
 } logsoftc;
 
-int	log_open;			/* also used in log() */
+/*
+ * log_mtx protects logsoftc, log_open.  Note that log_mtx does *not*
+ * protect the structures associated with msgbuf, which require Giant.
+ */
+struct mtx	log_mtx;
+int		log_open;		/* also used in log() */
 
 /* Times per second to check for a pending syslog wakeup. */
 static int	log_wakeups_per_second = 5;
@@ -98,17 +103,24 @@
 static	int
 logopen(dev_t dev, int flags, int mode, struct thread *td)
 {
-	if (log_open)
+
+	mtx_lock(&log_mtx);
+	if (log_open) {
+		mtx_unlock(&log_mtx);
 		return (EBUSY);
+	}
 	log_open = 1;
-	callout_init(&logsoftc.sc_callout, 0);
+	callout_init(&logsoftc.sc_callout, CALLOUT_MPSAFE);
+	mtx_unlock(&log_mtx);
 	fsetown(td->td_proc->p_pid, &logsoftc.sc_sigio);	/* signal process only */
+	mtx_lock(&log_mtx);
 	if (log_wakeups_per_second < 1) {
 		printf("syslog wakeup is less than one.  Adjusting to 1.\n");
 		log_wakeups_per_second = 1;
 	}
 	callout_reset(&logsoftc.sc_callout, hz / log_wakeups_per_second,
 	    logtimeout, NULL);
+	mtx_unlock(&log_mtx);
 	return (0);
 }
 
@@ -117,9 +129,11 @@
 logclose(dev_t dev, int flag, int mode, struct thread *td)
 {
 
+	mtx_lock(&log_mtx);
 	log_open = 0;
 	callout_stop(&logsoftc.sc_callout);
 	logsoftc.sc_state = 0;
+	mtx_unlock(&log_mtx);
 	funsetown(&logsoftc.sc_sigio);
 	return (0);
 }
@@ -138,14 +152,18 @@
 			splx(s);
 			return (EWOULDBLOCK);
 		}
+		mtx_lock(&log_mtx);
 		logsoftc.sc_state |= LOG_RDWAIT;
+		mtx_unlock(&log_mtx);
 		if ((error = tsleep(mbp, LOG_RDPRI | PCATCH, "klog", 0))) {
 			splx(s);
 			return (error);
 		}
 	}
 	splx(s);
+	mtx_lock(&log_mtx);
 	logsoftc.sc_state &= ~LOG_RDWAIT;
+	mtx_unlock(&log_mtx);
 
 	while (uio->uio_resid > 0) {
 		l = imin(sizeof(buf), uio->uio_resid);
@@ -182,8 +200,11 @@
 logtimeout(void *arg)
 {
 
-	if (!log_open)
+	mtx_lock(&log_mtx);
+	if (!log_open) {
+		mtx_unlock(&log_mtx);
 		return;
+	}
 	if (log_wakeups_per_second < 1) {
 		printf("syslog wakeup is less than one.  Adjusting to 1.\n");
 		log_wakeups_per_second = 1;
@@ -191,6 +212,7 @@
 	if (msgbuftrigger == 0) {
 		callout_reset(&logsoftc.sc_callout,
 		    hz / log_wakeups_per_second, logtimeout, NULL);
+		mtx_unlock(&log_mtx);
 		return;
 	}
 	msgbuftrigger = 0;
@@ -203,6 +225,7 @@
 	}
 	callout_reset(&logsoftc.sc_callout, hz / log_wakeups_per_second,
 	    logtimeout, NULL);
+	mtx_unlock(&log_mtx);
 }
 
 /*ARGSUSED*/
@@ -221,10 +244,12 @@
 		break;
 
 	case FIOASYNC:
+		mtx_lock(&log_mtx);
 		if (*(int *)data)
 			logsoftc.sc_state |= LOG_ASYNC;
 		else
 			logsoftc.sc_state &= ~LOG_ASYNC;
+		mtx_unlock(&log_mtx);
 		break;
 
 	case FIOSETOWN:
@@ -253,6 +278,7 @@
 log_drvinit(void *unused)
 {
 
+	mtx_init(&log_mtx, "log_mtx", NULL, MTX_DEF);
 	make_dev(&log_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "klog");
 }
 
--- //depot/vendor/freebsd/src/sys/kern/subr_prf.c	2004/02/18 21:30:32
+++ //depot/user/rwatson/netperf/sys/kern/subr_prf.c	2004/03/06 09:01:33
@@ -87,6 +87,9 @@
 	size_t	remain;
 };
 
+/*
+ * XXXRW: We access subr_log.c's log_open variable unlocked.
+ */
 extern	int log_open;
 
 static void  msglogchar(int c, int pri);
--- //depot/vendor/freebsd/src/sys/kern/sys_socket.c	2003/06/18 11:20:36
+++ //depot/user/rwatson/netperf/sys/kern/sys_socket.c	2004/03/01 16:13:52
@@ -79,16 +79,15 @@
 	struct socket *so = fp->f_data;
 	int error;
 
-	mtx_lock(&Giant);
 #ifdef MAC
+	/* XXX: Socket lock needed here? */
 	error = mac_check_socket_receive(active_cred, so);
-	if (error) {
-		mtx_unlock(&Giant);
+	if (error)
 		return (error);
-	}
 #endif
+	NET_LOCK_GIANT();
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, 0, uio, 0, 0, 0);
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -104,17 +103,16 @@
 	struct socket *so = fp->f_data;
 	int error;
 
-	mtx_lock(&Giant);
 #ifdef MAC
+	/* XXX: Socket lock needed here? */
 	error = mac_check_socket_send(active_cred, so);
-	if (error) {
-		mtx_unlock(&Giant);
+	if (error)
 		return (error);
-	}
 #endif
+	NET_LOCK_GIANT();
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, 0, uio, 0, 0, 0,
 						    uio->uio_td);
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
--- //depot/vendor/freebsd/src/sys/kern/uipc_domain.c	2003/09/02 14:00:52
+++ //depot/user/rwatson/netperf/sys/kern/uipc_domain.c	2004/02/29 20:52:43
@@ -134,8 +134,13 @@
 	if (max_linkhdr < 16)		/* XXX */
 		max_linkhdr = 16;
 
-	callout_init(&pffast_callout, 0);
-	callout_init(&pfslow_callout, 0);
+	if (debug_mpsafenet) {
+		callout_init(&pffast_callout, CALLOUT_MPSAFE);
+		callout_init(&pfslow_callout, CALLOUT_MPSAFE);
+	} else {
+		callout_init(&pffast_callout, 0);
+		callout_init(&pfslow_callout, 0);
+	}
 
 	callout_reset(&pffast_callout, 1, pffasttimo, NULL);
 	callout_reset(&pfslow_callout, 1, pfslowtimo, NULL);
--- //depot/vendor/freebsd/src/sys/kern/uipc_socket.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/kern/uipc_socket.c	2004/02/29 21:00:26
@@ -145,6 +145,8 @@
 			return so;
 		}
 #endif
+		SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd");
+		SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv");
 		/* XXX race condition for reentrant kernel */
 		so->so_gencnt = ++so_gencnt;
 		/* sx_init(&so->so_sxlock, "socket sxlock"); */
@@ -203,6 +205,7 @@
 	soref(so);
 	error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
 	if (error) {
+		SOCK_LOCK(so);
 		so->so_state |= SS_NOFDREF;
 		sorele(so);
 		return (error);
@@ -246,6 +249,8 @@
 	mac_destroy_socket(so);
 #endif
 	crfree(so->so_cred);
+	SOCKBUF_LOCK_DESTROY(&so->so_snd);
+	SOCKBUF_LOCK_DESTROY(&so->so_rcv);
 	/* sx_destroy(&so->so_sxlock); */
 	uma_zfree(socket_zone, so);
 	--numopensockets;
@@ -270,11 +275,13 @@
 		splx(s);
 		return (error);
 	}
+	SOCKBUF_LOCK(&so->so_rcv);
 	if (TAILQ_EMPTY(&so->so_comp))
 		so->so_options |= SO_ACCEPTCONN;
 	if (backlog < 0 || backlog > somaxconn)
 		backlog = somaxconn;
 	so->so_qlimit = backlog;
+	SOCKBUF_UNLOCK(&so->so_rcv);
 	splx(s);
 	return (0);
 }
@@ -283,14 +290,17 @@
 sofree(so)
 	struct socket *so;
 {
-	struct socket *head = so->so_head;
-	int s;
-
 	KASSERT(so->so_count == 0, ("socket %p so_count not 0", so));
+	SOCK_LOCK_ASSERT(so);
 
-	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0)
+	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
+		SOCK_UNLOCK(so);
 		return;
-	if (head != NULL) {
+	}
+	SOCK_UNLOCK(so);
+	SOCKBUF_LOCK(&so->so_rcv);
+	if (so->so_head != NULL) {
+		struct socket *head = so->so_head;
 		if (so->so_state & SS_INCOMP) {
 			TAILQ_REMOVE(&head->so_incomp, so, so_list);
 			head->so_incqlen--;
@@ -301,6 +311,7 @@
 			 * accept(2) may hang after select(2) indicated
 			 * that the listening socket was ready.
 			 */
+			/* XXX SOCKBUF_UNLOCK(&so->so_rcv); */
 			return;
 		} else {
 			panic("sofree: not queued");
@@ -308,13 +319,14 @@
 		so->so_state &= ~SS_INCOMP;
 		so->so_head = NULL;
 	}
+	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_flags |= SB_NOINTR;
 	(void)sblock(&so->so_snd, M_WAITOK);
-	s = splimp();
-	socantsendmore(so);
-	splx(s);
+	socantsendmore_locked(so);
 	sbunlock(&so->so_snd);
 	sbrelease(&so->so_snd, so);
+	SOCKBUF_UNLOCK(&so->so_snd);
 	sorflush(so);
 	sodealloc(so);
 }
@@ -354,11 +366,14 @@
 			(void) soabort(sp);
 		}
 	}
+	SOCK_LOCK(so);
 	if (so->so_pcb == 0)
 		goto discard;
 	if (so->so_state & SS_ISCONNECTED) {
 		if ((so->so_state & SS_ISDISCONNECTING) == 0) {
+			SOCK_UNLOCK(so);
 			error = sodisconnect(so);
+			SOCK_LOCK(so);
 			if (error)
 				goto drop;
 		}
@@ -367,7 +382,7 @@
 			    (so->so_state & SS_NBIO))
 				goto drop;
 			while (so->so_state & SS_ISCONNECTED) {
-				error = tsleep(&so->so_timeo,
+				error = msleep(&so->so_timeo, SOCK_MTX(so),
 				    PSOCK | PCATCH, "soclos", so->so_linger * hz);
 				if (error)
 					break;
@@ -376,7 +391,10 @@
 	}
 drop:
 	if (so->so_pcb) {
-		int error2 = (*so->so_proto->pr_usrreqs->pru_detach)(so);
+		int error2;
+		SOCK_UNLOCK(so);
+		error2 = (*so->so_proto->pr_usrreqs->pru_detach)(so);
+		SOCK_LOCK(so);
 		if (error == 0)
 			error = error2;
 	}
@@ -400,6 +418,7 @@
 
 	error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
 	if (error) {
+		SOCK_LOCK(so);
 		sotryfree(so);	/* note: does not decrement the ref count */
 		return error;
 	}
@@ -411,14 +430,12 @@
 	struct socket *so;
 	struct sockaddr **nam;
 {
-	int s = splnet();
 	int error;
 
 	if ((so->so_state & SS_NOFDREF) == 0)
 		panic("soaccept: !NOFDREF");
 	so->so_state &= ~SS_NOFDREF;
 	error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
-	splx(s);
 	return (error);
 }
 
@@ -428,12 +445,10 @@
 	struct sockaddr *nam;
 	struct thread *td;
 {
-	int s;
 	int error;
 
 	if (so->so_options & SO_ACCEPTCONN)
 		return (EOPNOTSUPP);
-	s = splnet();
 	/*
 	 * If protocol is connection-based, can only connect once.
 	 * Otherwise, if connected, try to disconnect first.
@@ -446,7 +461,6 @@
 		error = EISCONN;
 	else
 		error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td);
-	splx(s);
 	return (error);
 }
 
@@ -455,11 +469,9 @@
 	struct socket *so1;
 	struct socket *so2;
 {
-	int s = splnet();
 	int error;
 
 	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
-	splx(s);
 	return (error);
 }
 
@@ -467,20 +479,13 @@
 sodisconnect(so)
 	struct socket *so;
 {
-	int s = splnet();
 	int error;
 
-	if ((so->so_state & SS_ISCONNECTED) == 0) {
-		error = ENOTCONN;
-		goto bad;
-	}
-	if (so->so_state & SS_ISDISCONNECTING) {
-		error = EALREADY;
-		goto bad;
-	}
+	if ((so->so_state & SS_ISCONNECTED) == 0)
+		return ENOTCONN;
+	if (so->so_state & SS_ISDISCONNECTING)
+		return EALREADY;
 	error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
-bad:
-	splx(s);
 	return (error);
 }
 
@@ -531,7 +536,7 @@
 	struct mbuf **mp;
 	struct mbuf *m;
 	long space, len, resid;
-	int clen = 0, error, s, dontroute, mlen;
+	int clen = 0, error, dontroute, mlen;
 	int atomic = sosendallatonce(so) || top;
 #ifdef ZERO_COPY_SOCKETS
 	int cow_send;
@@ -563,20 +568,18 @@
 		td->td_proc->p_stats->p_ru.ru_msgsnd++;
 	if (control)
 		clen = control->m_len;
-#define	snderr(errno)	{ error = (errno); splx(s); goto release; }
+#define	snderr(errno)	{ error = (errno); goto release; }
 
-restart:
+	SOCKBUF_LOCK(&so->so_snd);
 	error = sblock(&so->so_snd, SBLOCKWAIT(flags));
 	if (error)
 		goto out;
 	do {
-		s = splnet();
 		if (so->so_state & SS_CANTSENDMORE)
 			snderr(EPIPE);
 		if (so->so_error) {
 			error = so->so_error;
 			so->so_error = 0;
-			splx(s);
 			goto release;
 		}
 		if ((so->so_state & SS_ISCONNECTED) == 0) {
@@ -605,14 +608,11 @@
 		    (atomic || space < so->so_snd.sb_lowat || space < clen)) {
 			if (so->so_state & SS_NBIO)
 				snderr(EWOULDBLOCK);
-			sbunlock(&so->so_snd);
 			error = sbwait(&so->so_snd);
-			splx(s);
 			if (error)
-				goto out;
-			goto restart;
+				goto release;
+			continue;
 		}
-		splx(s);
 		mp = &top;
 		space -= clen;
 		do {
@@ -627,10 +627,12 @@
 #ifdef ZERO_COPY_SOCKETS
 			cow_send = 0;
 #endif /* ZERO_COPY_SOCKETS */
+			SOCKBUF_UNLOCK(&so->so_snd);
 			if (top == 0) {
 				MGETHDR(m, M_TRYWAIT, MT_DATA);
 				if (m == NULL) {
 					error = ENOBUFS;
+					SOCKBUF_LOCK(&so->so_snd); /* XXX */
 					goto release;
 				}
 				mlen = MHLEN;
@@ -640,6 +642,7 @@
 				MGET(m, M_TRYWAIT, MT_DATA);
 				if (m == NULL) {
 					error = ENOBUFS;
+					SOCKBUF_LOCK(&so->so_snd); /* XXX */
 					goto release;
 				}
 				mlen = MLEN;
@@ -687,6 +690,7 @@
 			else
 #endif /* ZERO_COPY_SOCKETS */
 			error = uiomove(mtod(m, void *), (int)len, uio);
+			SOCKBUF_LOCK(&so->so_snd);
 			resid = uio->uio_resid;
 			m->m_len = len;
 			*mp = m;
@@ -702,13 +706,12 @@
 		    } while (space > 0 && atomic);
 		    if (dontroute)
 			    so->so_options |= SO_DONTROUTE;
-		    s = splnet();				/* XXX */
 		    /*
 		     * XXX all the SS_CANTSENDMORE checks previously
 		     * done could be out of date.  We could have recieved
 		     * a reset packet in an interrupt or maybe we slept
 		     * while doing page faults in uiomove() etc. We could
-		     * probably recheck again inside the splnet() protection
+		     * probably recheck again inside the locking protection
 		     * here, but there are probably other places that this
 		     * also happens.  We must rethink this.
 		     */
@@ -726,7 +729,6 @@
 			/* If there is more to send set PRUS_MORETOCOME */
 			(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
 			top, addr, control, td);
-		    splx(s);
 		    if (dontroute)
 			    so->so_options &= ~SO_DONTROUTE;
 		    clen = 0;
@@ -741,6 +743,7 @@
 release:
 	sbunlock(&so->so_snd);
 out:
+	SOCKBUF_UNLOCK(&so->so_snd);
 	if (top)
 		m_freem(top);
 	if (control)
@@ -774,7 +777,7 @@
 	int *flagsp;
 {
 	struct mbuf *m, **mp;
-	int flags, len, error, s, offset;
+	int flags, len, error, offset;
 	struct protosw *pr = so->so_proto;
 	struct mbuf *nextrecord;
 	int moff, type = 0;
@@ -832,12 +835,12 @@
 	if (so->so_state & SS_ISCONFIRMING && uio->uio_resid)
 		(*pr->pr_usrreqs->pru_rcvd)(so, 0);
 
-restart:
+	SOCKBUF_LOCK(&so->so_rcv);
 	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
 	if (error)
-		return (error);
-	s = splnet();
+		goto out;
 
+restart:
 	m = so->so_rcv.sb_mb;
 	/*
 	 * If we have less data than requested, block awaiting more
@@ -855,9 +858,8 @@
 	    (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
 	    ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
 	    m->m_nextpkt == 0 && (pr->pr_flags & PR_ATOMIC) == 0)) {
-		KASSERT(m != 0 || !so->so_rcv.sb_cc,
-		    ("receive: m == %p so->so_rcv.sb_cc == %u",
-		    m, so->so_rcv.sb_cc));
+		KASSERT(!(m == 0 && so->so_rcv.sb_cc),
+		    ("m %p so->so_rcv.sb_cc %u", m, so->so_rcv.sb_cc));
 		if (so->so_error) {
 			if (m)
 				goto dontblock;
@@ -890,14 +892,13 @@
 		}
 		SBLASTRECORDCHK(&so->so_rcv);
 		SBLASTMBUFCHK(&so->so_rcv);
-		sbunlock(&so->so_rcv);
 		error = sbwait(&so->so_rcv);
-		splx(s);
 		if (error)
-			return (error);
+			goto release;
 		goto restart;
 	}
 dontblock:
+	KASSERT(error == 0, ("unexpected state, error %u", error));
 	if (uio->uio_td)
 		uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++;
 	SBLASTRECORDCHK(&so->so_rcv);
@@ -906,10 +907,14 @@
 	if (pr->pr_flags & PR_ADDR) {
 		KASSERT(m->m_type == MT_SONAME,
 		    ("m->m_type == %d", m->m_type));
-		orig_resid = 0;
-		if (psa)
+		if (psa) {
 			*psa = sodupsockaddr(mtod(m, struct sockaddr *),
-			    mp0 == NULL ? M_WAITOK : M_NOWAIT);
+					M_NOWAIT);
+			if (*psa == NULL) {
+				error = ENOMEM;
+				goto release;
+			}
+		}
 		if (flags & MSG_PEEK) {
 			m = m->m_next;
 		} else {
@@ -917,30 +922,56 @@
 			so->so_rcv.sb_mb = m_free(m);
 			m = so->so_rcv.sb_mb;
 		}
+		orig_resid = 0;
 	}
-	while (m && m->m_type == MT_CONTROL && error == 0) {
-		if (flags & MSG_PEEK) {
-			if (controlp)
-				*controlp = m_copy(m, 0, m->m_len);
-			m = m->m_next;
-		} else {
-			sbfree(&so->so_rcv, m);
-			so->so_rcv.sb_mb = m->m_next;
-			m->m_next = NULL;
-			if (pr->pr_domain->dom_externalize)
-				error =
-				(*pr->pr_domain->dom_externalize)(m, controlp);
-			else if (controlp)
-				*controlp = m;
-			else
-				m_freem(m);
-			m = so->so_rcv.sb_mb;
+	if (m && m->m_type == MT_CONTROL) {
+		struct mbuf *cm = NULL;
+		struct mbuf **cme = &cm;
+
+		do {
+			if (flags & MSG_PEEK) {
+				if (controlp) {
+					SOCKBUF_UNLOCK(&so->so_rcv);
+					*controlp = m_copym(m, 0, m->m_len,
+						M_TRYWAIT);
+					SOCKBUF_LOCK(&so->so_rcv);
+					if (*controlp == NULL) {
+						error = ENOBUFS;
+						goto release;
+					}
+					controlp = &(*controlp)->m_next;
+				}
+				m = m->m_next;
+			} else {
+				sbfree(&so->so_rcv, m);
+				so->so_rcv.sb_mb = m->m_next;
+				m->m_next = NULL;
+				if (controlp) {
+					/*
+					 * Collect mbufs for processing below.
+					 */
+					*cme = m;
+					cme = &(*cme)->m_next;
+				} else
+					m_free(m);
+				m = so->so_rcv.sb_mb;
+			}
+		} while (m && m->m_type == MT_CONTROL);
+		if (cm != NULL) {
+			if (pr->pr_domain->dom_externalize) {
+				/*
+				 * NB: drop the lock to avoid potential LORs;
+				 * in particular unix domain sockets grab the
+				 * file descriptor lock which would be a LOR.
+				 */
+				SOCKBUF_UNLOCK(&so->so_rcv);
+				error = (*pr->pr_domain->dom_externalize)
+						(cm, controlp);
+				SOCKBUF_LOCK(&so->so_rcv);
+			} else
+				m_freem(cm);
 		}
-		if (controlp) {
-			orig_resid = 0;
-			while (*controlp != NULL)
-				controlp = &(*controlp)->m_next;
-		}
+		orig_resid = 0;
 	}
 	if (m) {
 		if ((flags & MSG_PEEK) == 0) {
@@ -997,7 +1028,7 @@
 		if (mp == 0) {
 			SBLASTRECORDCHK(&so->so_rcv);
 			SBLASTMBUFCHK(&so->so_rcv);
-			splx(s);
+			SOCKBUF_UNLOCK(&so->so_rcv);
 #ifdef ZERO_COPY_SOCKETS
 			if (so_zero_copy_receive) {
 				vm_page_t pg;
@@ -1021,7 +1052,7 @@
 			} else
 #endif /* ZERO_COPY_SOCKETS */
 			error = uiomove(mtod(m, char *) + moff, (int)len, uio);
-			s = splnet();
+			SOCKBUF_LOCK(&so->so_rcv);
 			if (error)
 				goto release;
 		} else
@@ -1102,9 +1133,8 @@
 			SBLASTMBUFCHK(&so->so_rcv);
 			error = sbwait(&so->so_rcv);
 			if (error) {
-				sbunlock(&so->so_rcv);
-				splx(s);
-				return (0);
+				error = 0;
+				goto release;
 			}
 			m = so->so_rcv.sb_mb;
 			if (m)
@@ -1137,17 +1167,15 @@
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 	}
 	if (orig_resid == uio->uio_resid && orig_resid &&
-	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
-		sbunlock(&so->so_rcv);
-		splx(s);
-		goto restart;
-	}
+	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0)
+		goto restart;		/* XXX multi-counts msgs */
 
 	if (flagsp)
 		*flagsp |= flags;
 release:
 	sbunlock(&so->so_rcv);
-	splx(s);
+out:
+	SOCKBUF_UNLOCK(&so->so_rcv);
 	return (error);
 }
 
@@ -1174,22 +1202,21 @@
 {
 	struct sockbuf *sb = &so->so_rcv;
 	struct protosw *pr = so->so_proto;
-	int s;
 	struct sockbuf asb;
 
+	SOCKBUF_LOCK(sb);
 	sb->sb_flags |= SB_NOINTR;
 	(void) sblock(sb, M_WAITOK);
-	s = splimp();
-	socantrcvmore(so);
+	socantrcvmore_locked(so);
 	sbunlock(sb);
 	asb = *sb;
 	/*
-	 * Invalidate/clear most of the sockbuf structure, but keep
-	 * its selinfo structure valid.
+	 * Invalidate/clear most of the sockbuf structure, but leave
+	 * selinfo and mutex data unchanged.
 	 */
 	bzero(&sb->sb_startzero,
 	    sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
-	splx(s);
+	SOCKBUF_UNLOCK(sb);
 
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose)
 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
@@ -1207,6 +1234,7 @@
 	struct so_accf	*af = so->so_accf;
 	int	error = 0;
 
+/* XXX locking */
 	/* do not set/remove accept filters on non listen sockets */
 	if ((so->so_options & SO_ACCEPTCONN) == 0) {
 		error = EINVAL;
@@ -1803,7 +1831,6 @@
 {
 	struct socket *so = kn->kn_fp->f_data;
 	struct sockbuf *sb;
-	int s;
 
 	switch (kn->kn_filter) {
 	case EVFILT_READ:
@@ -1821,10 +1848,10 @@
 		return (1);
 	}
 
-	s = splnet();
+	SOCKBUF_LOCK(sb);
 	SLIST_INSERT_HEAD(&sb->sb_sel.si_note, kn, kn_selnext);
 	sb->sb_flags |= SB_KNOTE;
-	splx(s);
+	SOCKBUF_UNLOCK(sb);
 	return (0);
 }
 
@@ -1832,12 +1859,12 @@
 filt_sordetach(struct knote *kn)
 {
 	struct socket *so = kn->kn_fp->f_data;
-	int s = splnet();
 
+	SOCKBUF_LOCK(&so->so_rcv);
 	SLIST_REMOVE(&so->so_rcv.sb_sel.si_note, kn, knote, kn_selnext);
 	if (SLIST_EMPTY(&so->so_rcv.sb_sel.si_note))
 		so->so_rcv.sb_flags &= ~SB_KNOTE;
-	splx(s);
+	SOCKBUF_UNLOCK(&so->so_rcv);
 }
 
 /*ARGSUSED*/
@@ -1845,30 +1872,35 @@
 filt_soread(struct knote *kn, long hint)
 {
 	struct socket *so = kn->kn_fp->f_data;
+	int result;
 
+	SOCKBUF_LOCK(&so->so_rcv);	/* XXX too conservative? */
 	kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
 	if (so->so_state & SS_CANTRCVMORE) {
 		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
-		return (1);
+		result = 1;
+	} else if (so->so_error) {	/* temporary udp error */
+		result = 1;
+	} else if (kn->kn_sfflags & NOTE_LOWAT) {
+		result = (kn->kn_data >= kn->kn_sdata);
+	} else {
+		result = (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat);
 	}
-	if (so->so_error)	/* temporary udp error */
-		return (1);
-	if (kn->kn_sfflags & NOTE_LOWAT)
-		return (kn->kn_data >= kn->kn_sdata);
-	return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat);
+	SOCKBUF_UNLOCK(&so->so_rcv);
+	return (result);
 }
 
 static void
 filt_sowdetach(struct knote *kn)
 {
 	struct socket *so = kn->kn_fp->f_data;
-	int s = splnet();
 
+	SOCKBUF_LOCK(&so->so_snd);
 	SLIST_REMOVE(&so->so_snd.sb_sel.si_note, kn, knote, kn_selnext);
 	if (SLIST_EMPTY(&so->so_snd.sb_sel.si_note))
 		so->so_snd.sb_flags &= ~SB_KNOTE;
-	splx(s);
+	SOCKBUF_UNLOCK(&so->so_snd);
 }
 
 /*ARGSUSED*/
@@ -1876,21 +1908,26 @@
 filt_sowrite(struct knote *kn, long hint)
 {
 	struct socket *so = kn->kn_fp->f_data;
+	int result;
 
+	SOCKBUF_LOCK(&so->so_snd);	/* XXX too conservative? */
 	kn->kn_data = sbspace(&so->so_snd);
 	if (so->so_state & SS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
-		return (1);
+		result = 1;
+	} else if (so->so_error) {	/* temporary udp error */
+		result = 1;
+	} else if (((so->so_state & SS_ISCONNECTED) == 0) &&
+	    (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
+		result = 0;
+	} else if (kn->kn_sfflags & NOTE_LOWAT) {
+		result = (kn->kn_data >= kn->kn_sdata);
+	} else {
+		result = (kn->kn_data >= so->so_snd.sb_lowat);
 	}
-	if (so->so_error)	/* temporary udp error */
-		return (1);
-	if (((so->so_state & SS_ISCONNECTED) == 0) &&
-	    (so->so_proto->pr_flags & PR_CONNREQUIRED))
-		return (0);
-	if (kn->kn_sfflags & NOTE_LOWAT)
-		return (kn->kn_data >= kn->kn_sdata);
-	return (kn->kn_data >= so->so_snd.sb_lowat);
+	SOCKBUF_UNLOCK(&so->so_snd);
+	return (result);
 }
 
 /*ARGSUSED*/
--- //depot/vendor/freebsd/src/sys/kern/uipc_socket2.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/kern/uipc_socket2.c	2004/02/29 21:00:26
@@ -108,9 +108,14 @@
 soisconnecting(so)
 	register struct socket *so;
 {
+	int need_lock = !SOCK_OWNED(so);
 
+	if (need_lock)
+		SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISCONNECTING;
+	if (need_lock)
+		SOCK_UNLOCK(so);
 }
 
 void
@@ -118,56 +123,79 @@
 	struct socket *so;
 {
 	struct socket *head = so->so_head;
+	int need_lock = !SOCK_OWNED(so);
 
+	if (need_lock)
+		SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
 	if (head && (so->so_state & SS_INCOMP)) {
-		if ((so->so_options & SO_ACCEPTFILTER) != 0) {
+		if ((so->so_options & SO_ACCEPTFILTER) == 0) {
+			if (need_lock)
+				SOCK_UNLOCK(so);
+			SOCK_LOCK(head);
+			TAILQ_REMOVE(&head->so_incomp, so, so_list);
+			head->so_incqlen--;
+			so->so_state &= ~SS_INCOMP;
+			TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
+			head->so_qlen++;
+			so->so_state |= SS_COMP;
+			sorwakeup_locked(head);
+			wakeup_one(&head->so_timeo);
+			SOCK_UNLOCK(head);
+		} else {
+/* XXX locking */
 			so->so_upcall = head->so_accf->so_accept_filter->accf_callback;
 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
 			so->so_rcv.sb_flags |= SB_UPCALL;
 			so->so_options &= ~SO_ACCEPTFILTER;
 			so->so_upcall(so, so->so_upcallarg, M_TRYWAIT);
-			return;
 		}
-		TAILQ_REMOVE(&head->so_incomp, so, so_list);
-		head->so_incqlen--;
-		so->so_state &= ~SS_INCOMP;
-		TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
-		head->so_qlen++;
-		so->so_state |= SS_COMP;
-		sorwakeup(head);
-		wakeup_one(&head->so_timeo);
 	} else {
 		wakeup(&so->so_timeo);
+		SOCK_UNLOCK(so);
 		sorwakeup(so);
 		sowwakeup(so);
 	}
+	if (!need_lock)
+		SOCK_LOCK(so);
 }
 
 void
 soisdisconnecting(so)
 	register struct socket *so;
 {
+	int need_lock = !SOCK_OWNED(so);
 
+	if (need_lock)
+		SOCK_LOCK(so);
 	so->so_state &= ~SS_ISCONNECTING;
 	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
 	wakeup(&so->so_timeo);
+	SOCK_UNLOCK(so);
 	sowwakeup(so);
 	sorwakeup(so);
+	if (!need_lock)
+		SOCK_LOCK(so);
 }
 
 void
 soisdisconnected(so)
 	register struct socket *so;
 {
+	int need_lock = !SOCK_OWNED(so);
 
+	if (need_lock)
+		SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
 	wakeup(&so->so_timeo);
+	SOCK_UNLOCK(so);
 	sbdrop(&so->so_snd, so->so_snd.sb_cc);
 	sowwakeup(so);
 	sorwakeup(so);
+	if (!need_lock)
+		SOCK_LOCK(so);
 }
 
 /*
@@ -186,8 +214,12 @@
 	int connstatus;
 {
 	register struct socket *so;
+	int over;
 
-	if (head->so_qlen > 3 * head->so_qlimit / 2)
+	SOCK_LOCK(head);
+	over = (head->so_qlen > 3 * head->so_qlimit / 2);
+	SOCK_UNLOCK(head);
+	if (over)
 		return ((struct socket *)0);
 	so = soalloc(M_NOWAIT);
 	if (so == NULL)
@@ -205,12 +237,13 @@
 #ifdef MAC
 	mac_create_socket_from_socket(head, so);
 #endif
+
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
 		sodealloc(so);
 		return ((struct socket *)0);
 	}
-
+	SOCKBUF_LOCK(&head->so_rcv);
 	if (connstatus) {
 		TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
 		so->so_state |= SS_COMP;
@@ -225,10 +258,11 @@
 		so->so_state |= SS_INCOMP;
 		head->so_incqlen++;
 	}
+	SOCKBUF_UNLOCK(&head->so_rcv);
 	if (connstatus) {
+		so->so_state |= connstatus;
 		sorwakeup(head);
 		wakeup(&head->so_timeo);
-		so->so_state |= connstatus;
 	}
 	return (so);
 }
@@ -253,6 +287,16 @@
 }
 
 void
+socantsendmore_locked(so)
+	struct socket *so;
+{
+	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+
+	so->so_state |= SS_CANTSENDMORE;
+	sowwakeup(so);
+}
+
+void
 socantrcvmore(so)
 	struct socket *so;
 {
@@ -261,6 +305,16 @@
 	sorwakeup(so);
 }
 
+void
+socantrcvmore_locked(so)
+	struct socket *so;
+{
+	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+
+	so->so_state |= SS_CANTRCVMORE;
+	sorwakeup_locked(so);
+}
+
 /*
  * Wait for data to arrive at/drain from a socket buffer.
  */
@@ -268,9 +322,10 @@
 sbwait(sb)
 	struct sockbuf *sb;
 {
+	SOCKBUF_LOCK_ASSERT(sb);
 
 	sb->sb_flags |= SB_WAIT;
-	return (tsleep(&sb->sb_cc,
+	return (msleep(&sb->sb_cc, &sb->sb_mtx,
 	    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
 	    sb->sb_timeo));
 }
@@ -285,9 +340,11 @@
 {
 	int error;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	while (sb->sb_flags & SB_LOCK) {
 		sb->sb_flags |= SB_WANT;
-		error = tsleep(&sb->sb_flags,
+		error = msleep(&sb->sb_flags, &sb->sb_mtx,
 		    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH,
 		    "sblock", 0);
 		if (error)
@@ -298,29 +355,71 @@
 }
 
 /*
+ * The part of sowakeup that must be done while
+ * holding the sockbuf lock.
+ */
+static __inline void
+sowakeup_under_lock(struct socket *so, struct sockbuf *sb)
+{
+	SOCKBUF_LOCK_ASSERT(sb);
+
+	selwakeuppri(&sb->sb_sel, PSOCK);
+	sb->sb_flags &= ~SB_SEL;
+	if (sb->sb_flags & SB_WAIT) {
+		sb->sb_flags &= ~SB_WAIT;
+		wakeup(&sb->sb_cc);
+	}
+}
+
+/*
  * Wakeup processes waiting on a socket buffer.
  * Do asynchronous notification via SIGIO
  * if the socket has the SS_ASYNC flag set.
+ *
+ * The caller is assumed to hold the necessary
+ * sockbuf lock.
  */
 void
+sowakeup_locked(so, sb)
+	register struct socket *so;
+	register struct sockbuf *sb;
+{
+
+	sowakeup_under_lock(so, sb);
+
+	if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
+		pgsigio(&so->so_sigio, SIGIO, 0);
+	if (sb->sb_flags & SB_UPCALL)
+		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
+	if (sb->sb_flags & SB_AIO)		/* XXX locking */
+		aio_swake(so, sb);
+	KNOTE(&sb->sb_sel.si_note, 0);		/* XXX locking? */
+}
+
+/*
+ * Wakeup processes waiting on a socket buffer.
+ * Do asynchronous notification via SIGIO
+ * if the socket has the SS_ASYNC flag set.
+ *
+ * The caller does not hold the sockbuf lock.
+ */
+void
 sowakeup(so, sb)
 	register struct socket *so;
 	register struct sockbuf *sb;
 {
 
-	selwakeuppri(&sb->sb_sel, PSOCK);
-	sb->sb_flags &= ~SB_SEL;
-	if (sb->sb_flags & SB_WAIT) {
-		sb->sb_flags &= ~SB_WAIT;
-		wakeup(&sb->sb_cc);
-	}
+	SOCKBUF_LOCK(sb);
+	sowakeup_under_lock(so, sb);
+	SOCKBUF_UNLOCK(sb);
+
 	if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
 		pgsigio(&so->so_sigio, SIGIO, 0);
 	if (sb->sb_flags & SB_UPCALL)
 		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
-	if (sb->sb_flags & SB_AIO)
+	if (sb->sb_flags & SB_AIO)		/* XXX locking */
 		aio_swake(so, sb);
-	KNOTE(&sb->sb_sel.si_note, 0);
+	KNOTE(&sb->sb_sel.si_note, 0);		/* XXX locking? */
 }
 
 /*
@@ -480,6 +579,8 @@
 {
 	struct mbuf *m = sb->sb_mb;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	while (m && m->m_nextpkt)
 		m = m->m_nextpkt;
 
@@ -499,6 +600,8 @@
 	struct mbuf *m = sb->sb_mb;
 	struct mbuf *n;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	while (m && m->m_nextpkt)
 		m = m->m_nextpkt;
 
@@ -535,7 +638,7 @@
  * discarded and mbufs are compacted where possible.
  */
 void
-sbappend(sb, m)
+sbappend_locked(sb, m)
 	struct sockbuf *sb;
 	struct mbuf *m;
 {
@@ -543,6 +646,9 @@
 
 	if (m == 0)
 		return;
+
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	SBLASTRECORDCHK(sb);
 	n = sb->sb_mb;
 	if (n) {
@@ -550,7 +656,7 @@
 			n = n->m_nextpkt;
 		do {
 			if (n->m_flags & M_EOR) {
-				sbappendrecord(sb, m); /* XXXXXX!!!! */
+				sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
 				return;
 			}
 		} while (n->m_next && (n = n->m_next));
@@ -563,7 +669,7 @@
 		if ((n = sb->sb_lastrecord) != NULL) {
 			do {
 				if (n->m_flags & M_EOR) {
-					sbappendrecord(sb, m); /* XXXXXX!!!! */
+					sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
 					return;
 				}
 			} while (n->m_next && (n = n->m_next));
@@ -580,13 +686,33 @@
 }
 
 /*
+ * Append mbuf chain m to the last record in the
+ * socket buffer sb.  The additional space associated
+ * the mbuf chain is recorded in sb.  Empty mbufs are
+ * discarded and mbufs are compacted where possible.
+ */
+void
+sbappend(sb, m)
+	struct sockbuf *sb;
+	struct mbuf *m;
+{
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		sbappend_locked(sb, m);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		sbappend_locked(sb, m);
+}
+
+/*
  * This version of sbappend() should only be used when the caller
  * absolutely knows that there will never be more than one record
  * in the socket buffer, that is, a stream protocol (such as TCP).
  */
 void
-sbappendstream(struct sockbuf *sb, struct mbuf *m)
+sbappendstream_locked(struct sockbuf *sb, struct mbuf *m)
 {
+	SOCKBUF_LOCK_ASSERT(sb);
 
 	KASSERT(m->m_nextpkt == NULL,("sbappendstream 0"));
 	KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1"));
@@ -596,13 +722,28 @@
 #ifdef MBUFTRACE
 	m_claim(m, sb->sb_mowner);
 #endif
-
 	sbcompress(sb, m, sb->sb_mbtail);
 
 	sb->sb_lastrecord = sb->sb_mb;
 	SBLASTRECORDCHK(sb);
 }
 
+/*
+ * This version of sbappend() should only be used when the caller
+ * absolutely knows that there will never be more than one record
+ * in the socket buffer, that is, a stream protocol (such as TCP).
+ */
+void
+sbappendstream(struct sockbuf *sb, struct mbuf *m)
+{
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		sbappendstream_locked(sb, m);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		sbappendstream_locked(sb, m);
+}
+
 #ifdef SOCKBUF_DEBUG
 void
 sbcheck(sb)
@@ -612,6 +753,8 @@
 	struct mbuf *n = 0;
 	u_long len = 0, mbcnt = 0;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	for (m = sb->sb_mb; m; m = n) {
 	    n = m->m_nextpkt;
 	    for (; m; m = m->m_next) {
@@ -634,12 +777,14 @@
  * begins a new record.
  */
 void
-sbappendrecord(sb, m0)
+sbappendrecord_locked(sb, m0)
 	register struct sockbuf *sb;
 	register struct mbuf *m0;
 {
 	register struct mbuf *m;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	if (m0 == 0)
 		return;
 	m = sb->sb_mb;
@@ -667,18 +812,37 @@
 }
 
 /*
+ * As above, except the mbuf chain
+ * begins a new record.
+ */
+void
+sbappendrecord(sb, m0)
+	register struct sockbuf *sb;
+	register struct mbuf *m0;
+{
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		sbappendrecord_locked(sb, m0);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		sbappendrecord_locked(sb, m0);
+}
+
+/*
  * As above except that OOB data
  * is inserted at the beginning of the sockbuf,
  * but after any other OOB data.
  */
 void
-sbinsertoob(sb, m0)
+sbinsertoob_locked(sb, m0)
 	register struct sockbuf *sb;
 	register struct mbuf *m0;
 {
 	register struct mbuf *m;
 	register struct mbuf **mp;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	if (m0 == 0)
 		return;
 	for (mp = &sb->sb_mb; *mp ; mp = &((*mp)->m_nextpkt)) {
@@ -713,13 +877,31 @@
 }
 
 /*
+ * As above except that OOB data
+ * is inserted at the beginning of the sockbuf,
+ * but after any other OOB data.
+ */
+void
+sbinsertoob(sb, m0)
+	register struct sockbuf *sb;
+	register struct mbuf *m0;
+{
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		sbinsertoob_locked(sb, m0);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		sbinsertoob_locked(sb, m0);
+}
+
+/*
  * Append address and data, and optionally, control (ancillary) data
  * to the receive queue of a socket.  If present,
  * m0 must include a packet header with total length.
  * Returns 0 if no space in sockbuf or insufficient mbufs.
  */
 int
-sbappendaddr(sb, asa, m0, control)
+sbappendaddr_locked(sb, asa, m0, control)
 	struct sockbuf *sb;
 	struct sockaddr *asa;
 	struct mbuf *m0, *control;
@@ -727,11 +909,14 @@
 	struct mbuf *m, *n, *nlast;
 	int space = asa->sa_len;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	if (m0 && (m0->m_flags & M_PKTHDR) == 0)
 		panic("sbappendaddr");
 	if (m0)
 		space += m0->m_pkthdr.len;
 	space += m_length(control, &n);
+
 	if (space > sbspace(sb))
 		return (0);
 #if MSIZE <= 256
@@ -753,25 +938,50 @@
 	sballoc(sb, n);
 	nlast = n;
 	SBLINKRECORD(sb, m);
+	sb->sb_mbtail = nlast;
 
-	sb->sb_mbtail = nlast;
 	SBLASTMBUFCHK(sb);
-
 	SBLASTRECORDCHK(sb);
 	return (1);
 }
 
+/*
+ * Append address and data, and optionally, control (ancillary) data
+ * to the receive queue of a socket.  If present,
+ * m0 must include a packet header with total length.
+ * Returns 0 if no space in sockbuf or insufficient mbufs.
+ */
+int
+sbappendaddr(sb, asa, m0, control)
+	struct sockbuf *sb;
+	struct sockaddr *asa;
+	struct mbuf *m0, *control;
+{
+	int retval;
+
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		retval = sbappendaddr_locked(sb, asa, m0, control);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		retval = sbappendaddr_locked(sb, asa, m0, control);
+	return (retval);
+}
+
 int
-sbappendcontrol(sb, m0, control)
+sbappendcontrol_locked(sb, m0, control)
 	struct sockbuf *sb;
 	struct mbuf *control, *m0;
 {
 	struct mbuf *m, *n, *mlast;
 	int space;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	if (control == 0)
 		panic("sbappendcontrol");
 	space = m_length(control, &n) + m_length(m0, NULL);
+
 	if (space > sbspace(sb))
 		return (0);
 	n->m_next = m0;			/* concatenate data to control */
@@ -783,14 +993,30 @@
 	sballoc(sb, m);
 	mlast = m;
 	SBLINKRECORD(sb, control);
+	sb->sb_mbtail = mlast;
 
-	sb->sb_mbtail = mlast;
 	SBLASTMBUFCHK(sb);
+	SBLASTRECORDCHK(sb);
 
-	SBLASTRECORDCHK(sb);
 	return (1);
 }
 
+int
+sbappendcontrol(sb, m0, control)
+	struct sockbuf *sb;
+	struct mbuf *control, *m0;
+{
+	int retval;
+
+	if (!SOCKBUF_OWNED(sb)) {
+		SOCKBUF_LOCK(sb);
+		retval = sbappendcontrol(sb, m0, control);
+		SOCKBUF_UNLOCK(sb);
+	} else
+		retval = sbappendcontrol(sb, m0, control);
+	return (retval);
+}
+
 /*
  * Compress mbuf chain m into the socket
  * buffer sb following mbuf n.  If n
@@ -804,6 +1030,8 @@
 	register int eor = 0;
 	register struct mbuf *o;
 
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	while (m) {
 		eor |= m->m_flags & M_EOR;
 		if (m->m_len == 0 &&
@@ -872,7 +1100,8 @@
 		sbdrop(sb, (int)sb->sb_cc);
 	}
 	if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt)
-		panic("sbflush: cc %u || mb %p || mbcnt %u", sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt);
+		panic("sbflush: cc %u || mb %p || mbcnt %u",
+			sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt);
 }
 
 /*
@@ -885,7 +1114,11 @@
 {
 	register struct mbuf *m;
 	struct mbuf *next;
+	int need_lock = !SOCKBUF_OWNED(sb);
 
+	if (need_lock)
+		SOCKBUF_LOCK(sb);
+
 	next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
 	while (len > 0) {
 		if (m == 0) {
@@ -929,6 +1162,9 @@
 	} else if (m->m_nextpkt == NULL) {
 		sb->sb_lastrecord = m;
 	}
+
+	if (need_lock)
+		SOCKBUF_UNLOCK(sb);
 }
 
 /*
@@ -940,7 +1176,11 @@
 	register struct sockbuf *sb;
 {
 	register struct mbuf *m;
+	int need_lock = !SOCKBUF_OWNED(sb);
 
+	if (need_lock)
+		SOCKBUF_LOCK(sb);
+
 	m = sb->sb_mb;
 	if (m) {
 		sb->sb_mb = m->m_nextpkt;
@@ -950,6 +1190,9 @@
 		} while (m);
 	}
 	SB_EMPTY_FIXUP(sb);
+
+	if (need_lock)
+		SOCKBUF_UNLOCK(sb);
 }
 
 /*
--- //depot/vendor/freebsd/src/sys/kern/uipc_syscalls.c	2004/03/03 18:00:46
+++ //depot/user/rwatson/netperf/sys/kern/uipc_syscalls.c	2004/03/03 21:04:46
@@ -117,10 +117,10 @@
 	if (error)
 		return (error);
 	/* An extra reference on `fp' has been held for us by falloc(). */
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	error = socreate(uap->domain, &so, uap->type, uap->protocol,
 	    td->td_ucred, td);
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	FILEDESC_LOCK(fdp);
 	if (error) {
 		if (fdp->fd_ofiles[fd] == fp) {
@@ -174,10 +174,11 @@
 	struct socket *so;
 	int error;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, fd, &so, NULL)) != 0)
 		goto done2;
 #ifdef MAC
+	/* XXXRW: MAC requires socket lock? */
 	error = mac_check_socket_bind(td->td_ucred, so, sa);
 	if (error)
 		goto done1;
@@ -188,7 +189,7 @@
 #endif
 	fputsock(so);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	FREE(sa, M_SONAME);
 	return (error);
 }
@@ -208,9 +209,10 @@
 	struct socket *so;
 	int error;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 #ifdef MAC
+		/* XXXRW: MAC requires socket lock? */
 		error = mac_check_socket_listen(td->td_ucred, so);
 		if (error)
 			goto done;
@@ -221,7 +223,7 @@
 #endif
 		fputsock(so);
 	}
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -260,12 +262,14 @@
 			goto done3;
 		}
 	}
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	error = fgetsock(td, uap->s, &head, &fflag);
 	if (error)
 		goto done2;
 	s = splnet();
+	SOCK_LOCK(head);
 	if ((head->so_options & SO_ACCEPTCONN) == 0) {
+		SOCK_UNLOCK(head);
 		splx(s);
 		error = EINVAL;
 		goto done;
@@ -279,9 +283,10 @@
 			head->so_error = EWOULDBLOCK;
 			break;
 		}
-		error = tsleep(&head->so_timeo, PSOCK | PCATCH,
+		error = msleep(&head->so_timeo, SOCK_MTX(head), PSOCK | PCATCH,
 		    "accept", 0);
 		if (error) {
+			SOCK_UNLOCK(head);
 			splx(s);
 			goto done;
 		}
@@ -289,6 +294,7 @@
 	if (head->so_error) {
 		error = head->so_error;
 		head->so_error = 0;
+		SOCK_UNLOCK(head);
 		splx(s);
 		goto done;
 	}
@@ -298,11 +304,13 @@
 	 * ready to be accepted. Remove it from the queue prior to
 	 * allocating the file descriptor for it since falloc() may
 	 * block allowing another process to accept the connection
-	 * instead.
+	 * instead.  The reference previously owned by the socket queue
+	 * is now thread-local, letting us release the lock on the head.
 	 */
 	so = TAILQ_FIRST(&head->so_comp);
 	TAILQ_REMOVE(&head->so_comp, so, so_list);
 	head->so_qlen--;
+	SOCK_UNLOCK(head);
 
 	error = falloc(td, &nfp, &fd);
 	if (error) {
@@ -310,20 +318,27 @@
 		 * Probably ran out of file descriptors. Put the
 		 * unaccepted connection back onto the queue and
 		 * do another wakeup so some other process might
-		 * have a chance at it.
+		 * have a chance at it.  Note that strict ordering
+		 * is lost.
 		 */
+		SOCK_LOCK(head);
 		TAILQ_INSERT_HEAD(&head->so_comp, so, so_list);
 		head->so_qlen++;
 		wakeup_one(&head->so_timeo);
+		SOCK_UNLOCK(head);
 		splx(s);
 		goto done;
 	}
 	/* An extra reference on `nfp' has been held for us by falloc(). */
 	td->td_retval[0] = fd;
 
+	/* XXX lock? */
 	/* connection has been removed from the listen queue */
 	KNOTE(&head->so_rcv.sb_sel.si_note, 0);
 
+	/*
+	 * XXXRW: so should be locked to modify so_state here?
+	 */
 	so->so_state &= ~SS_COMP;
 	so->so_head = NULL;
 	pgid = fgetown(&head->so_sigio);
@@ -408,7 +423,7 @@
 		fdrop(nfp, td);
 	fputsock(head);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 done3:
 	return (error);
 }
@@ -473,7 +488,7 @@
 	int error, s;
 	int interrupted = 0;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, fd, &so, NULL)) != 0)
 		goto done2;
 	if (so->so_state & SS_ISCONNECTING) {
@@ -481,6 +496,7 @@
 		goto done1;
 	}
 #ifdef MAC
+	/* XXXRW: MAC requires socket lock? */
 	error = mac_check_socket_connect(td->td_ucred, so, sa);
 	if (error)
 		goto bad;
@@ -493,8 +509,10 @@
 		goto done1;
 	}
 	s = splnet();
+	SOCK_LOCK(so);
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		error = tsleep(&so->so_timeo, PSOCK | PCATCH, "connec", 0);
+		error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH,
+		    "connec", 0);
 		if (error) {
 			if (error == EINTR || error == ERESTART)
 				interrupted = 1;
@@ -505,6 +523,7 @@
 		error = so->so_error;
 		so->so_error = 0;
 	}
+	SOCK_UNLOCK(so);
 	splx(s);
 bad:
 	if (!interrupted)
@@ -514,7 +533,7 @@
 done1:
 	fputsock(so);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	FREE(sa, M_SONAME);
 	return (error);
 }
@@ -537,7 +556,7 @@
 	struct socket *so1, *so2;
 	int fd, error, sv[2];
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	error = socreate(uap->domain, &so1, uap->type, uap->protocol,
 	    td->td_ucred, td);
 	if (error)
@@ -609,7 +628,7 @@
 free1:
 	(void)soclose(so1);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -695,11 +714,12 @@
 	int iovlen;
 #endif
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		goto bad2;
 
 #ifdef MAC
+	/* XXXRW: MAC requires socket lock? */
 	error = mac_check_socket_send(td->td_ucred, so);
 	if (error)
 		goto bad;
@@ -756,7 +776,7 @@
 bad:
 	fputsock(so);
 bad2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -936,17 +956,17 @@
 	int iovlen;
 #endif
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, s, &so, NULL)) != 0) {
-		mtx_unlock(&Giant);
+		NET_UNLOCK_GIANT();
 		return (error);
 	}
 
 #ifdef MAC
+	/* XXXRW: MAC requires socket lock? */
 	error = mac_check_socket_receive(td->td_ucred, so);
 	if (error) {
 		fputsock(so);
-		mtx_unlock(&Giant);
 		return (error);
 	}
 #endif
@@ -1070,7 +1090,7 @@
 	}
 out:
 	fputsock(so);
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	if (fromsa)
 		FREE(fromsa, M_SONAME);
 	if (control)
@@ -1285,12 +1305,12 @@
 	struct socket *so;
 	int error;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 		error = soshutdown(so, uap->how);
 		fputsock(so);
 	}
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -1318,7 +1338,7 @@
 	if (uap->valsize < 0)
 		return (EINVAL);
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 		sopt.sopt_dir = SOPT_SET;
 		sopt.sopt_level = uap->level;
@@ -1329,7 +1349,7 @@
 		error = sosetopt(so, &sopt);
 		fputsock(so);
 	}
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -1353,7 +1373,7 @@
 	struct  socket *so;
 	struct	sockopt sopt;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
 		goto done2;
 	if (uap->val) {
@@ -1383,7 +1403,7 @@
 done1:
 	fputsock(so);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1408,7 +1428,7 @@
 	socklen_t len;
 	int error;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
 	error = copyin(uap->alen, &len, sizeof (len));
@@ -1442,7 +1462,7 @@
 done1:
 	fputsock(so);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1493,7 +1513,7 @@
 	socklen_t len;
 	int error;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
@@ -1532,7 +1552,7 @@
 done1:
 	fputsock(so);
 done2:
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1686,7 +1706,7 @@
 	int error, s, headersize = 0, headersent = 0;
 	struct iovec *hdr_iov = NULL;
 
-	mtx_lock(&Giant);
+	NET_LOCK_GIANT();
 
 	hdtr_size = 0;
 
@@ -1718,6 +1738,7 @@
 	}
 
 #ifdef MAC
+	/* XXXRW: MAC requires socket lock? */
 	error = mac_check_socket_send(td->td_ucred, so);
 	if (error)
 		goto done;
@@ -1757,7 +1778,9 @@
 	/*
 	 * Protect against multiple writers to the socket.
 	 */
+	SOCKBUF_LOCK(&so->so_snd);
 	(void) sblock(&so->so_snd, M_WAITOK);
+	SOCKBUF_UNLOCK(&so->so_snd);
 
 	/*
 	 * Loop through the pages in the file, starting with the requested
@@ -1791,14 +1814,17 @@
 		 * Optimize the non-blocking case by looking at the socket space
 		 * before going to the extra work of constituting the sf_buf.
 		 */
+		SOCKBUF_LOCK(&so->so_snd);
 		if ((so->so_state & SS_NBIO) && sbspace(&so->so_snd) <= 0) {
 			if (so->so_state & SS_CANTSENDMORE)
 				error = EPIPE;
 			else
 				error = EAGAIN;
 			sbunlock(&so->so_snd);
+			SOCKBUF_UNLOCK(&so->so_snd);
 			goto done;
 		}
+		SOCKBUF_UNLOCK(&so->so_snd);
 		VM_OBJECT_LOCK(obj);
 		/*
 		 * Attempt to look up the page.
@@ -1887,7 +1913,9 @@
 			}
 			vm_page_unlock_queues();
 			VM_OBJECT_UNLOCK(obj);
+			SOCKBUF_LOCK(&so->so_snd);
 			sbunlock(&so->so_snd);
+			SOCKBUF_UNLOCK(&so->so_snd);
 			goto done;
 		}
 		vm_page_unlock_queues();
@@ -1903,7 +1931,9 @@
 			if (pg->wire_count == 0 && pg->object == NULL)
 				vm_page_free(pg);
 			vm_page_unlock_queues();
+			SOCKBUF_LOCK(&so->so_snd);
 			sbunlock(&so->so_snd);
+			SOCKBUF_UNLOCK(&so->so_snd);
 			error = EINTR;
 			goto done;
 		}
@@ -1940,6 +1970,7 @@
 		 * Add the buffer to the socket buffer chain.
 		 */
 		s = splnet();
+		SOCKBUF_LOCK(&so->so_snd);
 retry_space:
 		/*
 		 * Make sure that the socket is still able to take more data.
@@ -1961,6 +1992,7 @@
 			}
 			m_freem(m);
 			sbunlock(&so->so_snd);
+			SOCKBUF_UNLOCK(&so->so_snd);
 			splx(s);
 			goto done;
 		}
@@ -1973,6 +2005,7 @@
 			if (so->so_state & SS_NBIO) {
 				m_freem(m);
 				sbunlock(&so->so_snd);
+				SOCKBUF_UNLOCK(&so->so_snd);
 				splx(s);
 				error = EAGAIN;
 				goto done;
@@ -1992,14 +2025,20 @@
 			goto retry_space;
 		}
 		error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, m, 0, 0, td);
+		/* XXX: Why release and re-grab? */
+		SOCKBUF_UNLOCK(&so->so_snd);
 		splx(s);
 		if (error) {
+			SOCKBUF_LOCK(&so->so_snd);
 			sbunlock(&so->so_snd);
+			SOCKBUF_UNLOCK(&so->so_snd);
 			goto done;
 		}
 		headersent = 1;
 	}
+	SOCKBUF_LOCK(&so->so_snd);
 	sbunlock(&so->so_snd);
+	SOCKBUF_UNLOCK(&so->so_snd);
 
 	/*
 	 * Send trailers. Wimp out and use writev(2).
@@ -2046,7 +2085,7 @@
 	if (m_header)
 		m_freem(m_header);
 
-	mtx_unlock(&Giant);
+	NET_UNLOCK_GIANT();
 
 	if (error == ERESTART)
 		error = EINTR;
--- //depot/vendor/freebsd/src/sys/kern/uipc_usrreq.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/kern/uipc_usrreq.c	2004/02/29 21:00:26
@@ -85,6 +85,39 @@
 static struct	sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL };
 static ino_t	unp_ino;		/* prototype for fake inode numbers */
 
+static struct mtx unp_mtx;
+#define	UNP_HEAD_LOCK_INIT() \
+	mtx_init(&unp_mtx, "unp head", NULL, MTX_DEF)
+#define	UNP_HEAD_LOCK()			mtx_lock(&unp_mtx)
+#define	UNP_HEAD_UNLOCK()		mtx_unlock(&unp_mtx)
+#define	UNP_HEAD_LOCK_ASSERT()		mtx_assert(&unp_mtx, MA_OWNED)
+
+/* NB: DUPOK is to cover the connect2 case XXX */
+#define	UNP_LOCK_INIT(_unp) \
+	mtx_init(&(_unp)->unp_mtx, "unp", NULL, MTX_DEF | MTX_DUPOK)
+#define	UNP_LOCK_DESTROY(_unp)		mtx_destroy(&(_unp)->unp_mtx)
+#define	UNP_LOCK(_unp)			mtx_lock(&(_unp)->unp_mtx)
+#define	UNP_UNLOCK(_unp)		mtx_unlock(&(_unp)->unp_mtx)
+#define	UNP_LOCK_ASSERT(_unp)		mtx_assert(&(_unp)->unp_mtx, MA_OWNED)
+
+/*
+ * A unp lock is always preceded by locking the head.
+ * Since this occurs often we define convenienc macros
+ * for entry, exist, and validation in lower-level routines.
+ */
+#define	UNP_ENTER(_unp) do {		\
+	UNP_HEAD_LOCK();		\
+	UNP_LOCK(_unp);			\
+} while (0)
+#define	UNP_EXIT(_unp) do {		\
+	UNP_UNLOCK(_unp);		\
+	UNP_HEAD_UNLOCK();		\
+} while (0)
+#define	UNP_ASSERT(_unp) do {		\
+	UNP_HEAD_LOCK_ASSERT();		\
+	UNP_LOCK_ASSERT(_unp);		\
+} while (0)
+
 static int     unp_attach(struct socket *);
 static void    unp_detach(struct unpcb *);
 static int     unp_bind(struct unpcb *,struct sockaddr *, struct thread *);
@@ -107,8 +140,10 @@
 
 	if (unp == 0)
 		return (EINVAL);
+	UNP_ENTER(unp);
 	unp_drop(unp, ECONNABORTED);
-	unp_detach(unp);
+	unp_detach(unp);	/* NB: unlocks unp + head */
+	SOCK_LOCK(so);
 	sotryfree(so);
 	return (0);
 }
@@ -117,6 +152,7 @@
 uipc_accept(struct socket *so, struct sockaddr **nam)
 {
 	struct unpcb *unp = sotounpcb(so);
+	struct sockaddr *sa;
 
 	if (unp == 0)
 		return (EINVAL);
@@ -126,13 +162,14 @@
 	 * if it was bound and we are still connected
 	 * (our peer may have closed already!).
 	 */
-	if (unp->unp_conn && unp->unp_conn->unp_addr) {
-		*nam = sodupsockaddr(
-		    (struct sockaddr *)unp->unp_conn->unp_addr, M_WAITOK);
-	} else {
-		*nam = sodupsockaddr((struct sockaddr *)&sun_noname,
-		    M_WAITOK);
-	}
+	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
+	UNP_ENTER(unp);
+	if (unp->unp_conn && unp->unp_conn->unp_addr)
+		sa = (struct sockaddr *) unp->unp_conn->unp_addr;
+	else
+		sa = &sun_noname;
+	bcopy(sa, *nam, sa->sa_len);
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -153,7 +190,6 @@
 
 	if (unp == 0)
 		return (EINVAL);
-
 	return (unp_bind(unp, nam, td));
 }
 
@@ -161,21 +197,30 @@
 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
 	struct unpcb *unp = sotounpcb(so);
+	int retval;
 
-	if (unp == 0)
-		return (EINVAL);
-	return (unp_connect(so, nam, curthread));
+	if (unp != 0) {
+		UNP_ENTER(unp);
+		retval = unp_connect(so, nam, curthread);
+		UNP_EXIT(unp);
+	} else
+		retval = EINVAL;
+	return (retval);
 }
 
 static int
 uipc_connect2(struct socket *so1, struct socket *so2)
 {
 	struct unpcb *unp = sotounpcb(so1);
+	int retval;
 
-	if (unp == 0)
-		return (EINVAL);
-
-	return (unp_connect2(so1, so2));
+	if (unp != 0) {
+		UNP_ENTER(unp);
+		retval = unp_connect2(so1, so2);
+		UNP_EXIT(unp);
+	} else
+		retval = EINVAL;
+	return (retval);
 }
 
 /* control is EOPNOTSUPP */
@@ -187,8 +232,8 @@
 
 	if (unp == 0)
 		return (EINVAL);
-
-	unp_detach(unp);
+	UNP_ENTER(unp);
+	unp_detach(unp);	/* NB: unlocks unp + head */
 	return (0);
 }
 
@@ -197,41 +242,52 @@
 {
 	struct unpcb *unp = sotounpcb(so);
 
-	if (unp == 0)
+	if (unp != 0) {
+		UNP_ENTER(unp);
+		unp_disconnect(unp);
+		UNP_EXIT(unp);
+		return (0);
+	} else
 		return (EINVAL);
-	unp_disconnect(unp);
-	return (0);
 }
 
 static int
 uipc_listen(struct socket *so, struct thread *td)
 {
 	struct unpcb *unp = sotounpcb(so);
+	int retval;
 
-	if (unp == 0 || unp->unp_vnode == 0)
+	if (unp != 0 && unp->unp_vnode != 0) {
+		UNP_ENTER(unp);
+		retval = unp_listen(unp, td);
+		UNP_EXIT(unp);
+	} else
 		return (EINVAL);
-	return (unp_listen(unp, td));
+	return (retval);
 }
 
 static int
 uipc_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	struct unpcb *unp = sotounpcb(so);
+	struct sockaddr *sa;
 
 	if (unp == 0)
 		return (EINVAL);
+	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
+	UNP_ENTER(unp);
 	if (unp->unp_conn && unp->unp_conn->unp_addr)
-		*nam = sodupsockaddr(
-		    (struct sockaddr *)unp->unp_conn->unp_addr, M_WAITOK);
+		sa = (struct sockaddr *) unp->unp_conn->unp_addr;
 	else {
 		/*
 		 * XXX: It seems that this test always fails even when
 		 * connection is established.  So, this else clause is
 		 * added as workaround to return PF_LOCAL sockaddr.
 		 */
-		*nam = sodupsockaddr((struct sockaddr *)&sun_noname,
-		    M_WAITOK);
+		sa = &sun_noname;
 	}
+	bcopy(sa, *nam, sa->sa_len);
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -244,15 +300,27 @@
 
 	if (unp == 0)
 		return (EINVAL);
+	/*
+	 * Reorder locks to avoid LORs.  Note that we
+	 * delay re-locking so_rcv to below so it can
+	 * be done only once.
+	 */
+	SOCKBUF_UNLOCK(&so->so_rcv);
+	UNP_ENTER(unp);
 	switch (so->so_type) {
 	case SOCK_DGRAM:
 		panic("uipc_rcvd DGRAM?");
 		/*NOTREACHED*/
 
 	case SOCK_STREAM:
-		if (unp->unp_conn == 0)
+		if (unp->unp_conn == 0) {
+			SOCKBUF_LOCK(&so->so_rcv);
 			break;
+		}
 		so2 = unp->unp_conn->unp_socket;
+		/* NB: careful of order here */
+		SOCKBUF_LOCK(&so2->so_snd);
+		SOCKBUF_LOCK(&so->so_rcv);
 		/*
 		 * Adjust backpressure on sender
 		 * and wakeup any waiting to write.
@@ -264,12 +332,14 @@
 		(void)chgsbsize(so2->so_cred->cr_uidinfo, &so2->so_snd.sb_hiwat,
 		    newhiwat, RLIM_INFINITY);
 		unp->unp_cc = so->so_rcv.sb_cc;
-		sowwakeup(so2);
+		sowwakeup_locked(so2);
+		SOCKBUF_UNLOCK(&so2->so_snd);
 		break;
 
 	default:
 		panic("uipc_rcvd unknown socktype");
 	}
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -296,6 +366,13 @@
 	if (control && (error = unp_internalize(&control, td)))
 		goto release;
 
+	/*
+	 * Reorder locks to avoid LORs.
+	 */
+	SOCKBUF_UNLOCK(&so->so_snd);
+	UNP_ENTER(unp);
+	SOCKBUF_LOCK(&so->so_snd);
+
 	switch (so->so_type) {
 	case SOCK_DGRAM:
 	{
@@ -306,7 +383,9 @@
 				error = EISCONN;
 				break;
 			}
+			SOCKBUF_UNLOCK(&so->so_snd);
 			error = unp_connect(so, nam, td);
+			SOCKBUF_LOCK(&so->so_snd);
 			if (error)
 				break;
 		} else {
@@ -320,13 +399,14 @@
 			from = (struct sockaddr *)unp->unp_addr;
 		else
 			from = &sun_noname;
-		if (sbappendaddr(&so2->so_rcv, from, m, control)) {
-			sorwakeup(so2);
+		SOCKBUF_LOCK(&so2->so_rcv);
+		if (sbappendaddr_locked(&so2->so_rcv, from, m, control)) {
+			sorwakeup_locked(so2);
 			m = 0;
 			control = 0;
-		} else {
+		} else
 			error = ENOBUFS;
-		}
+		SOCKBUF_UNLOCK(&so2->so_rcv);
 		if (nam)
 			unp_disconnect(unp);
 		break;
@@ -340,7 +420,9 @@
 		 */
 		if ((so->so_state & SS_ISCONNECTED) == 0) {
 			if (nam) {
+				SOCKBUF_UNLOCK(&so->so_snd);
 				error = unp_connect(so, nam, td);
+				SOCKBUF_LOCK(&so->so_snd);
 				if (error)
 					break;	/* XXX */
 			} else {
@@ -356,17 +438,17 @@
 		if (unp->unp_conn == 0)
 			panic("uipc_send connected but no connection?");
 		so2 = unp->unp_conn->unp_socket;
+		SOCKBUF_LOCK(&so2->so_rcv);
 		/*
 		 * Send to paired receive port, and then reduce
 		 * send buffer hiwater marks to maintain backpressure.
 		 * Wake up readers.
 		 */
 		if (control) {
-			if (sbappendcontrol(&so2->so_rcv, m, control))
+			if (sbappendcontrol_locked(&so2->so_rcv, m, control))
 				control = 0;
-		} else {
-			sbappend(&so2->so_rcv, m);
-		}
+		} else
+			sbappend_locked(&so2->so_rcv, m);
 		so->so_snd.sb_mbmax -=
 			so2->so_rcv.sb_mbcnt - unp->unp_conn->unp_mbcnt;
 		unp->unp_conn->unp_mbcnt = so2->so_rcv.sb_mbcnt;
@@ -375,7 +457,8 @@
 		(void)chgsbsize(so->so_cred->cr_uidinfo, &so->so_snd.sb_hiwat,
 		    newhiwat, RLIM_INFINITY);
 		unp->unp_conn->unp_cc = so2->so_rcv.sb_cc;
-		sorwakeup(so2);
+		sorwakeup_locked(so2);
+		SOCKBUF_UNLOCK(&so2->so_rcv);
 		m = 0;
 		break;
 
@@ -388,13 +471,13 @@
 	 * a SHUTDOWN.
 	 */
 	if (flags & PRUS_EOF) {
-		socantsendmore(so);
+		socantsendmore_locked(so);
 		unp_shutdown(unp);
 	}
+	UNP_EXIT(unp);
 
 	if (control && error != 0)
-		unp_dispose(control);
-
+		unp_dispose(control);		/* XXX need head lock? */
 release:
 	if (control)
 		m_freem(control);
@@ -411,6 +494,7 @@
 
 	if (unp == 0)
 		return (EINVAL);
+	UNP_ENTER(unp);
 	sb->st_blksize = so->so_snd.sb_hiwat;
 	if (so->so_type == SOCK_STREAM && unp->unp_conn != 0) {
 		so2 = unp->unp_conn->unp_socket;
@@ -420,6 +504,7 @@
 	if (unp->unp_ino == 0)
 		unp->unp_ino = (++unp_ino == 0) ? ++unp_ino : unp_ino;
 	sb->st_ino = unp->unp_ino;
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -430,8 +515,11 @@
 
 	if (unp == 0)
 		return (EINVAL);
+	UNP_ENTER(unp);
+	/* XXX socket lock? */
 	socantsendmore(so);
 	unp_shutdown(unp);
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -439,15 +527,18 @@
 uipc_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	struct unpcb *unp = sotounpcb(so);
+	struct sockaddr *sa;
 
 	if (unp == 0)
 		return (EINVAL);
+	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
+	UNP_ENTER(unp);
 	if (unp->unp_addr)
-		*nam = sodupsockaddr((struct sockaddr *)unp->unp_addr,
-		    M_WAITOK);
+		sa = (struct sockaddr *) unp->unp_addr;
 	else
-		*nam = sodupsockaddr((struct sockaddr *)&sun_noname,
-		    M_WAITOK);
+		sa = &sun_noname;
+	bcopy(sa, *nam, sa->sa_len);
+	UNP_EXIT(unp);
 	return (0);
 }
 
@@ -469,6 +560,7 @@
 
 	switch (sopt->sopt_dir) {
 	case SOPT_GET:
+		UNP_ENTER(unp);
 		switch (sopt->sopt_name) {
 		case LOCAL_PEERCRED:
 			if (unp->unp_flags & UNP_HAVEPC)
@@ -485,6 +577,7 @@
 			error = EOPNOTSUPP;
 			break;
 		}
+		UNP_EXIT(unp);
 		break;
 	case SOPT_SET:
 	default:
@@ -557,8 +650,13 @@
 	unp_count++;
 	LIST_INIT(&unp->unp_refs);
 	unp->unp_socket = so;
+	UNP_LOCK_INIT(unp);
+
+	UNP_HEAD_LOCK();
 	LIST_INSERT_HEAD(so->so_type == SOCK_DGRAM ? &unp_dhead
 			 : &unp_shead, unp, unp_link);
+	UNP_HEAD_UNLOCK();
+
 	so->so_pcb = unp;
 	return (0);
 }
@@ -567,18 +665,25 @@
 unp_detach(unp)
 	register struct unpcb *unp;
 {
+	struct vnode *vp;
+
+	UNP_ASSERT(unp);
+
 	LIST_REMOVE(unp, unp_link);
 	unp->unp_gencnt = ++unp_gencnt;
 	--unp_count;
-	if (unp->unp_vnode) {
+	if ((vp = unp->unp_vnode)) {
 		unp->unp_vnode->v_socket = 0;
-		vrele(unp->unp_vnode);
 		unp->unp_vnode = 0;
 	}
 	if (unp->unp_conn)
 		unp_disconnect(unp);
-	while (!LIST_EMPTY(&unp->unp_refs))
-		unp_drop(LIST_FIRST(&unp->unp_refs), ECONNRESET);
+	while (!LIST_EMPTY(&unp->unp_refs)) {
+		struct unpcb *ref = LIST_FIRST(&unp->unp_refs);
+		UNP_LOCK(ref);
+		unp_drop(ref, ECONNRESET);
+		UNP_UNLOCK(ref);
+	}
 	soisdisconnected(unp->unp_socket);
 	unp->unp_socket->so_pcb = 0;
 	if (unp_rights) {
@@ -594,7 +699,11 @@
 	}
 	if (unp->unp_addr)
 		FREE(unp->unp_addr, M_SONAME);
+	UNP_LOCK_DESTROY(unp);
+	UNP_HEAD_UNLOCK();
 	uma_zfree(unp_zone, unp);
+	if (vp)
+		vrele(vp);
 }
 
 static int
@@ -621,15 +730,14 @@
 	buf = malloc(namelen + 1, M_TEMP, M_WAITOK);
 	strlcpy(buf, soun->sun_path, namelen + 1);
 
+	mtx_lock(&Giant);
 restart:
 	NDINIT(&nd, CREATE, NOFOLLOW | LOCKPARENT | SAVENAME, UIO_SYSSPACE,
 	    buf, td);
 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
 	error = namei(&nd);
-	if (error) {
-		free(buf, M_TEMP);
-		return (error);
-	}
+	if (error)
+		goto done;
 	vp = nd.ni_vp;
 	if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
 		NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -639,14 +747,12 @@
 			vput(nd.ni_dvp);
 		if (vp != NULL) {
 			vrele(vp);
-			free(buf, M_TEMP);
-			return (EADDRINUSE);
+			error = EADDRINUSE;
+			goto done;
 		}
 		error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH);
-		if (error) {
-			free(buf, M_TEMP);
-			return (error);
-		}
+		if (error)
+			goto done;
 		goto restart;
 	}
 	VATTR_NULL(&vattr);
@@ -662,18 +768,21 @@
 	}
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	vput(nd.ni_dvp);
-	if (error) {
-		free(buf, M_TEMP);
-		return (error);
-	}
+	if (error)
+		goto done;
 	vp = nd.ni_vp;
+	soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK);
+	UNP_ENTER(unp);
 	vp->v_socket = unp->unp_socket;
 	unp->unp_vnode = vp;
-	unp->unp_addr = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK);
+	unp->unp_addr = soun;
+	UNP_EXIT(unp);
 	VOP_UNLOCK(vp, 0, td);
 	vn_finished_write(mp);
+done:
+	mtx_unlock(&Giant);
 	free(buf, M_TEMP);
-	return (0);
+	return (error);
 }
 
 static int
@@ -685,10 +794,14 @@
 	register struct sockaddr_un *soun = (struct sockaddr_un *)nam;
 	register struct vnode *vp;
 	register struct socket *so2, *so3;
-	struct unpcb *unp, *unp2, *unp3;
+	struct unpcb *unp = sotounpcb(so);
+	struct unpcb *unp2, *unp3;
 	int error, len;
 	struct nameidata nd;
 	char buf[SOCK_MAXADDRLEN];
+	struct sockaddr *sa;
+
+	UNP_ASSERT(unp);
 
 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
 	if (len <= 0)
@@ -696,9 +809,17 @@
 	strlcpy(buf, soun->sun_path, len + 1);
 
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, buf, td);
+	/* drop locks across namei */
+	UNP_EXIT(unp);
+	mtx_lock(&Giant);
 	error = namei(&nd);
+	if (!error)
+		sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
+	else
+		sa = NULL;
+	UNP_ENTER(unp);
 	if (error)
-		return (error);
+		goto bad2;
 	vp = nd.ni_vp;
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	if (vp->v_type != VSOCK) {
@@ -718,19 +839,30 @@
 		goto bad;
 	}
 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
-		if ((so2->so_options & SO_ACCEPTCONN) == 0 ||
-		    (so3 = sonewconn(so2, 0)) == 0) {
+		if (so2->so_options & SO_ACCEPTCONN) {
+			/*
+			 * NB: drop locks here so unp_attach is entered
+			 *     w/o locks; this avoids a recursive lock
+			 *     of the head and holding sleep locks across
+			 *     a (potentially) blocking malloc.
+			 */
+			UNP_EXIT(unp);
+			so3 = sonewconn(so2, 0);
+			UNP_ENTER(unp);
+		} else
+			so3 = 0;
+		if (so3 == 0) {
 			error = ECONNREFUSED;
 			goto bad;
 		}
 		unp = sotounpcb(so);
 		unp2 = sotounpcb(so2);
 		unp3 = sotounpcb(so3);
-		if (unp2->unp_addr)
-			unp3->unp_addr = (struct sockaddr_un *)
-			    sodupsockaddr((struct sockaddr *)unp2->unp_addr,
-			    M_WAITOK);
-
+		if (unp2->unp_addr) {
+			bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len);
+			unp3->unp_addr = (struct sockaddr_un *) sa;
+			sa = NULL;
+		}
 		/*
 		 * unp_peercred management:
 		 *
@@ -762,6 +894,10 @@
 	error = unp_connect2(so, so2);
 bad:
 	vput(vp);
+bad2:
+	if (sa)
+		free(sa, M_SONAME);
+	mtx_unlock(&Giant);
 	return (error);
 }
 
@@ -773,9 +909,12 @@
 	register struct unpcb *unp = sotounpcb(so);
 	register struct unpcb *unp2;
 
+	UNP_ASSERT(unp);
+
 	if (so2->so_type != so->so_type)
 		return (EPROTOTYPE);
 	unp2 = sotounpcb(so2);
+	UNP_LOCK(unp2);
 	unp->unp_conn = unp2;
 	switch (so->so_type) {
 
@@ -793,6 +932,7 @@
 	default:
 		panic("unp_connect2");
 	}
+	UNP_UNLOCK(unp2);
 	return (0);
 }
 
@@ -802,6 +942,8 @@
 {
 	register struct unpcb *unp2 = unp->unp_conn;
 
+	UNP_ASSERT(unp);
+
 	if (unp2 == 0)
 		return;
 	unp->unp_conn = 0;
@@ -814,10 +956,13 @@
 
 	case SOCK_STREAM:
 		soisdisconnected(unp->unp_socket);
+		UNP_LOCK(unp2);
 		unp2->unp_conn = 0;
 		soisdisconnected(unp2->unp_socket);
+		UNP_UNLOCK(unp2);
 		break;
 	}
+	return;
 }
 
 #ifdef notdef
@@ -860,8 +1005,10 @@
 	 * OK, now we're committed to doing something.
 	 */
 	xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK);
+	UNP_HEAD_LOCK();
 	gencnt = unp_gencnt;
 	n = unp_count;
+	UNP_HEAD_UNLOCK();
 
 	xug->xug_len = sizeof *xug;
 	xug->xug_count = n;
@@ -875,6 +1022,7 @@
 
 	unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK);
 
+	UNP_HEAD_LOCK();
 	for (unp = LIST_FIRST(head), i = 0; unp && i < n;
 	     unp = LIST_NEXT(unp, unp_link)) {
 		if (unp->unp_gencnt <= gencnt) {
@@ -884,6 +1032,7 @@
 			unp_list[i++] = unp;
 		}
 	}
+	UNP_HEAD_UNLOCK();
 	n = i;			/* in case we lost some during malloc */
 
 	error = 0;
@@ -941,6 +1090,8 @@
 {
 	struct socket *so;
 
+	UNP_ASSERT(unp);
+
 	if (unp->unp_socket->so_type == SOCK_STREAM && unp->unp_conn &&
 	    (so = unp->unp_conn->unp_socket))
 		socantrcvmore(so);
@@ -953,6 +1104,8 @@
 {
 	struct socket *so = unp->unp_socket;
 
+	UNP_ASSERT(unp);
+
 	so->so_error = errno;
 	unp_disconnect(unp);
 }
@@ -1104,6 +1257,8 @@
 	uma_zone_set_max(unp_zone, nmbclusters);
 	LIST_INIT(&unp_dhead);
 	LIST_INIT(&unp_shead);
+
+	UNP_HEAD_LOCK_INIT();
 }
 
 static int
@@ -1260,6 +1415,8 @@
 	struct file **extra_ref, **fpp;
 	int nunref, i;
 
+	UNP_HEAD_LOCK_ASSERT();		/* NB: this serializes entry */
+
 	if (unp_gcing)
 		return;
 	unp_gcing = 1;
@@ -1454,6 +1611,7 @@
 	struct unpcb *unp;
 	struct thread *td;
 {
+	UNP_ASSERT(unp);
 
 	cru2x(td->td_ucred, &unp->unp_peercred);
 	unp->unp_flags |= UNP_HAVEPCCACHED;
--- //depot/vendor/freebsd/src/sys/net/bpf.c	2004/02/29 07:35:30
+++ //depot/user/rwatson/netperf/sys/net/bpf.c	2004/03/01 16:13:52
@@ -557,7 +557,7 @@
 	struct ifnet *ifp;
 	struct mbuf *m;
 	int error;
-	static struct sockaddr dst;
+	struct sockaddr dst;
 	int datlen;
 
 	if (d->bd_bif == 0)
@@ -568,6 +568,7 @@
 	if (uio->uio_resid == 0)
 		return (0);
 
+	bzero(&dst, sizeof(dst));
 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, &m, &dst, &datlen);
 	if (error)
 		return (error);
@@ -583,12 +584,10 @@
 	mac_create_mbuf_from_bpfdesc(d, m);
 	BPFD_UNLOCK(d);
 #endif
-	mtx_lock(&Giant);
+	/* NB: the driver frees the mbuf */
+	NET_LOCK_GIANT();
 	error = (*ifp->if_output)(ifp, m, &dst, (struct rtentry *)0);
-	mtx_unlock(&Giant);
-	/*
-	 * The driver frees the mbuf.
-	 */
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
--- //depot/vendor/freebsd/src/sys/net/bpfdesc.h	2004/02/29 07:35:30
+++ //depot/user/rwatson/netperf/sys/net/bpfdesc.h	2004/02/29 22:51:24
@@ -102,7 +102,10 @@
 
 #define BPFD_LOCK(bd)		mtx_lock(&(bd)->bd_mtx)
 #define BPFD_UNLOCK(bd)		mtx_unlock(&(bd)->bd_mtx)
-#define BPFD_LOCK_ASSERT(bd)	mtx_assert(&(bd)->bd_mtx, MA_OWNED)
+#define BPFD_LOCK_ASSERT(bd)	do {					\
+	mtx_assert(&(bd)->bd_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 
 /* Test whether a BPF is ready for read(). */
 #define	bpf_ready(bd)						 \
--- //depot/vendor/freebsd/src/sys/net/if.c	2004/03/12 18:35:54
+++ //depot/user/rwatson/netperf/sys/net/if.c	2004/03/12 22:29:12
@@ -659,6 +659,7 @@
 
 /*
  * Create a clone network interface.
+ * XXXRW: Locking?
  */
 int
 if_clone_create(char *name, int len)
@@ -731,6 +732,7 @@
 
 /*
  * Destroy a clone network interface.
+ * XXXRW: Locking?
  */
 int
 if_clone_destroy(const char *name)
@@ -768,6 +770,7 @@
 
 /*
  * Look up a network interface cloner.
+ * XXXRW: Locking?
  */
 static struct if_clone *
 if_clone_lookup(const char *name, int *unitp)
@@ -809,6 +812,7 @@
 
 /*
  * Register a network interface cloner.
+ * XXXRW: Locking?
  */
 void
 if_clone_attach(struct if_clone *ifc)
@@ -851,6 +855,7 @@
 
 /*
  * Unregister a network interface cloner.
+ * XXXRW: Locking?
  */
 void
 if_clone_detach(struct if_clone *ifc)
@@ -863,6 +868,7 @@
 
 /*
  * Provide list of interface cloners to userspace.
+ * XXXRW: Locking?
  */
 static int
 if_clone_list(struct if_clonereq *ifcr)
--- //depot/vendor/freebsd/src/sys/net/if_gif.c	2003/12/27 20:00:55
+++ //depot/user/rwatson/netperf/sys/net/if_gif.c	2004/03/09 15:48:47
@@ -83,8 +83,14 @@
 
 #define GIFNAME		"gif"
 
+/*
+ * XXXRW: Note that gif_mtx only protects global gif-related data, not
+ * per-softc data.  See also netinet/in_gif.c for locking needs.
+ */
+static struct mtx gif_mtx;
 static MALLOC_DEFINE(M_GIF, "gif", "Generic Tunnel Interface");
 static LIST_HEAD(, gif_softc) gif_softc_list;
+static int gif_called = 0;
 
 void	(*ng_gif_input_p)(struct ifnet *ifp, struct mbuf **mp, int af);
 void	(*ng_gif_input_orphan_p)(struct ifnet *ifp, struct mbuf *m, int af);
@@ -145,7 +151,9 @@
 
 	gifattach0(sc);
 
+	mtx_lock(&gif_mtx);
 	LIST_INSERT_HEAD(&gif_softc_list, sc, gif_list);
+	mtx_unlock(&gif_mtx);
 	return (0);
 }
 
@@ -173,15 +181,14 @@
 		(*ng_gif_attach_p)(&sc->gif_if);
 }
 
-void
-gif_clone_destroy(ifp)
-	struct ifnet *ifp;
+static void
+gif_destroy(struct gif_softc *sc)
 {
+	struct ifnet *ifp = &sc->gif_if;
 	int err;
-	struct gif_softc *sc = ifp->if_softc;
+
+	gif_delete_tunnel(ifp);
 
-	gif_delete_tunnel(&sc->gif_if);
-	LIST_REMOVE(sc, gif_list);
 #ifdef INET6
 	if (sc->encap_cookie6 != NULL) {
 		err = encap_detach(sc->encap_cookie6);
@@ -203,15 +210,29 @@
 	free(sc, M_GIF);
 }
 
+void
+gif_clone_destroy(ifp)
+	struct ifnet *ifp;
+{
+	struct gif_softc *sc = ifp->if_softc;
+
+	mtx_lock(&gif_mtx);
+	LIST_REMOVE(sc, gif_list);
+	mtx_unlock(&gif_mtx);
+	gif_destroy(sc);
+}
+
 static int
 gifmodevent(mod, type, data)
 	module_t mod;
 	int type;
 	void *data;
 {
+	struct gif_softc *sc;
 
 	switch (type) {
 	case MOD_LOAD:
+		mtx_init(&gif_mtx, "gif_mtx", NULL, MTX_DEF);
 		LIST_INIT(&gif_softc_list);
 		if_clone_attach(&gif_cloner);
 
@@ -223,9 +244,15 @@
 	case MOD_UNLOAD:
 		if_clone_detach(&gif_cloner);
 
-		while (!LIST_EMPTY(&gif_softc_list))
-			gif_clone_destroy(&LIST_FIRST(&gif_softc_list)->gif_if);
-
+		mtx_lock(&gif_mtx);
+		while ((sc = LIST_FIRST(&gif_softc_list)) != NULL) {
+			LIST_REMOVE(sc, gif_list);
+			mtx_unlock(&gif_mtx);
+			gif_destroy(sc);
+			mtx_unlock(&gif_mtx);
+		}
+		mtx_unlock(&gif_mtx);
+		mtx_destroy(&gif_mtx);
 #ifdef INET6
 		ip6_gif_hlim = 0;
 #endif
@@ -314,7 +341,6 @@
 {
 	struct gif_softc *sc = (struct gif_softc*)ifp;
 	int error = 0;
-	static int called = 0;	/* XXX: MUTEX */
 
 #ifdef MAC
 	error = mac_check_ifnet_transmit(ifp, m);
@@ -330,15 +356,22 @@
 	 * XXX: this mechanism may introduce another problem about
 	 *      mutual exclusion of the variable CALLED, especially if we
 	 *      use kernel thread.
+	 * XXXRW: use atomic operations to improve performance.
+	 * XXXRW: This doesn't handle parallel entry into the gif code,
+	 *      since the counter is global and not packet-local.  Use an
+	 *      m_tag?
 	 */
-	if (++called > max_gif_nesting) {
+	mtx_lock(&gif_mtx);
+	if (++gif_called > max_gif_nesting) {
+		mtx_unlock(&gif_mtx);
 		log(LOG_NOTICE,
 		    "gif_output: recursively called too many times(%d)\n",
-		    called);
+		    gif_called);
 		m_freem(m);
 		error = EIO;	/* is there better errno? */
 		goto end;
 	}
+	mtx_unlock(&gif_mtx);
 
 	m->m_flags &= ~(M_BCAST|M_MCAST);
 	if (!(ifp->if_flags & IFF_UP) ||
@@ -378,7 +411,13 @@
 	}
 
   end:
-	called = 0;		/* reset recursion counter */
+	/*
+	 * This is completely bogus.  Shouldn't this be 'gif_called--',
+	 * and this is not MPSAFE.
+	 */
+	mtx_lock(&gif_mtx);
+	gif_called = 0;		/* reset recursion counter */
+	mtx_unlock(&gif_mtx);
 	if (error)
 		ifp->if_oerrors++;
 	return error;
@@ -451,6 +490,9 @@
 }
 
 /* XXX how should we handle IPv6 scope on SIOC[GS]IFPHYADDR? */
+/*
+ * XXXRW: per-gif softc locking required.
+ */
 int
 gif_ioctl(ifp, cmd, data)
 	struct ifnet *ifp;
@@ -688,6 +730,13 @@
 	return error;
 }
 
+/*
+ * XXXRW: There's a general event-ordering issue here: the code to check
+ * if a given tunnel is already present happens before we perform a
+ * potentially blocking setup of the tunnel.  This code needs to be
+ * re-ordered so that the check and replacement can be atomic using
+ * a mutex.
+ */
 int
 gif_set_tunnel(ifp, src, dst)
 	struct ifnet *ifp;
@@ -700,8 +749,10 @@
 	int s;
 	int error = 0; 
 
-	s = splnet();
-
+	/*
+	 * XXXRW: per-gif softc locking required.
+	 */
+	mtx_lock(&gif_mtx);
 	LIST_FOREACH(sc2, &gif_softc_list, gif_list) {
 		if (sc2 == sc)
 			continue;
@@ -721,12 +772,17 @@
 		    bcmp(sc2->gif_pdst, dst, dst->sa_len) == 0 &&
 		    bcmp(sc2->gif_psrc, src, src->sa_len) == 0) {
 			error = EADDRNOTAVAIL;
+			mtx_unlock(&gif_mtx);
 			goto bad;
 		}
 
 		/* XXX both end must be valid? (I mean, not 0.0.0.0) */
 	}
+	mtx_unlock(&gif_mtx);
 
+	/*
+	 * XXXRW: Lock gif softc fields.
+	 */
 	/* XXX we can detach from both, but be polite just in case */
 	if (sc->gif_psrc)
 		switch (sc->gif_psrc->sa_family) {
--- //depot/vendor/freebsd/src/sys/net/if_gre.c	2004/01/30 01:05:29
+++ //depot/user/rwatson/netperf/sys/net/if_gre.c	2004/03/11 22:56:06
@@ -91,6 +91,12 @@
 
 #define GRENAME	"gre"
 
+/*
+ * gre_mtx protects all global variables in if_gre.c.
+ *
+ * XXXRW: It does not protect softc-specific data.
+ */
+struct mtx gre_mtx;
 static MALLOC_DEFINE(M_GRE, GRENAME, "Generic Routing Encapsulation");
 
 struct gre_softc_head gre_softc_list;
@@ -149,6 +155,7 @@
 greattach(void)
 {
 
+	mtx_init(&gre_mtx, "gre_mtx", NULL, MTX_DEF);
 	LIST_INIT(&gre_softc_list);
 	if_clone_attach(&gre_cloner);
 }
@@ -181,24 +188,35 @@
 	sc->wccp_ver = WCCP_V1;
 	if_attach(&sc->sc_if);
 	bpfattach(&sc->sc_if, DLT_NULL, sizeof(u_int32_t));
+	mtx_lock(&gre_mtx);
 	LIST_INSERT_HEAD(&gre_softc_list, sc, sc_list);
+	mtx_unlock(&gre_mtx);
 	return (0);
 }
 
 static void
-gre_clone_destroy(ifp)
-	struct ifnet *ifp;
+gre_destroy(struct gre_softc *sc)
 {
-	struct gre_softc *sc = ifp->if_softc;
 
 #ifdef INET
 	if (sc->encap != NULL)
 		encap_detach(sc->encap);
 #endif
+	bpfdetach(&sc->sc_if);
+	if_detach(&sc->sc_if);
+	free(sc, M_GRE);
+}
+
+static void
+gre_clone_destroy(ifp)
+	struct ifnet *ifp;
+{
+	struct gre_softc *sc = ifp->if_softc;
+
+	mtx_lock(&gre_mtx);
 	LIST_REMOVE(sc, sc_list);
-	bpfdetach(ifp);
-	if_detach(ifp);
-	free(sc, M_GRE);
+	mtx_unlock(&gre_mtx);
+	gre_destroy(sc);
 }
 
 /*
@@ -727,6 +745,7 @@
 static int
 gremodevent(module_t mod, int type, void *data)
 {
+	struct gre_softc *sc;
 
 	switch (type) {
 	case MOD_LOAD:
@@ -735,8 +754,15 @@
 	case MOD_UNLOAD:
 		if_clone_detach(&gre_cloner);
 
-		while (!LIST_EMPTY(&gre_softc_list))
-			gre_clone_destroy(&LIST_FIRST(&gre_softc_list)->sc_if);
+		mtx_lock(&gre_mtx);
+		while ((sc = LIST_FIRST(&gre_softc_list)) != NULL) {
+			LIST_REMOVE(sc, sc_list);
+			mtx_unlock(&gre_mtx);
+			gre_destroy(sc);
+			mtx_lock(&gre_mtx);
+		}
+		mtx_unlock(&gre_mtx);
+		mtx_destroy(&gre_mtx);
 		break;
 	}
 	return 0;
--- //depot/vendor/freebsd/src/sys/net/if_gre.h	2004/01/26 04:35:31
+++ //depot/user/rwatson/netperf/sys/net/if_gre.h	2004/03/11 22:56:06
@@ -54,6 +54,12 @@
 	WCCP_V2
 } wccp_ver_t;
 
+/*
+ * XXXRW: softc fields need locking.
+ *
+ * XXXRW: gre's notion of a 'called' count is not MP-safe, as it assumes
+ * only one packet can be processed at a time.
+ */
 struct gre_softc {
 	struct ifnet sc_if;
 	LIST_ENTRY(gre_softc) sc_list;
@@ -176,6 +182,7 @@
 
 #ifdef _KERNEL
 LIST_HEAD(gre_softc_head, gre_softc);
+extern struct mtx gre_mtx;
 extern struct gre_softc_head gre_softc_list;
 
 u_int16_t	gre_in_cksum(u_int16_t *, u_int);
--- //depot/vendor/freebsd/src/sys/net/if_spppsubr.c	2004/03/12 22:21:11
+++ //depot/user/rwatson/netperf/sys/net/if_spppsubr.c	2004/03/11 23:46:14
@@ -17,7 +17,7 @@
  *
  * From: Version 2.4, Thu Apr 30 17:17:21 MSD 1997
  *
- * $FreeBSD: src/sys/net/if_spppsubr.c,v 1.106 2004/03/13 06:16:59 rwatson Exp $
+ * $FreeBSD: src/sys/net/if_spppsubr.c,v 1.105 2004/01/21 19:02:02 phk Exp $
  */
 
 #include <sys/param.h>
@@ -92,12 +92,8 @@
 #include <net/if_sppp.h>
 
 #if defined(__FreeBSD__) && __FreeBSD__ >= 3
-# define UNTIMEOUT(fun, arg, handle) untimeout(fun, arg, handle)
-# define TIMEOUT(fun, arg1, arg2, handle) handle = timeout(fun, arg1, arg2)
 # define IOCTL_CMD_T	u_long
 #else
-# define UNTIMEOUT(fun, arg, handle) untimeout(fun, arg)
-# define TIMEOUT(fun, arg1, arg2, handle) timeout(fun, arg1, arg2)
 # define IOCTL_CMD_T	int
 #endif
 
@@ -259,10 +255,11 @@
 	void	(*scr)(struct sppp *sp);
 };
 
+struct mtx sppp_mtx;
+MTX_SYSINIT(sppp_mtx, &sppp_mtx, "sppp_mtx", MTX_DEF);
+
 static struct sppp *spppq;
-#if defined(__FreeBSD__) && __FreeBSD__ >= 3
-static struct callout_handle keepalive_ch;
-#endif
+static struct callout keepalive_callout;
 
 #if defined(__FreeBSD__) && __FreeBSD__ >= 3 && __FreeBSD_version < 501113
 #define	SPP_FMT		"%s%d: "
@@ -960,13 +957,18 @@
 {
 	struct sppp *sp = (struct sppp*) ifp;
 
+	mtx_lock(&sppp_mtx);
 	/* Initialize keepalive handler. */
-	if (! spppq)
-		TIMEOUT(sppp_keepalive, 0, hz * 10, keepalive_ch);
+	if (spppq == NULL) {
+		callout_init(&keepalive_callout, 0);
+		callout_reset(&keepalive_callout, hz * 10, sppp_keepalive,
+		    NULL); 
+	}
 
 	/* Insert new entry into the keepalive list. */
 	sp->pp_next = spppq;
 	spppq = sp;
+	mtx_unlock(&sppp_mtx);
 
 	sp->pp_if.if_mtu = PP_MTU;
 	sp->pp_if.if_flags = IFF_POINTOPOINT | IFF_MULTICAST;
@@ -1012,6 +1014,7 @@
 	struct sppp **q, *p, *sp = (struct sppp*) ifp;
 	int i;
 
+	mtx_lock(&sppp_mtx);
 	/* Remove the entry from the keepalive list. */
 	for (q = &spppq; (p = *q); q = &p->pp_next)
 		if (p == sp) {
@@ -1020,12 +1023,13 @@
 		}
 
 	/* Stop keepalive handler. */
-	if (! spppq)
-		UNTIMEOUT(sppp_keepalive, 0, keepalive_ch);
+	if (spppq == NULL)
+		callout_stop(&keepalive_callout);
+	mtx_unlock(&sppp_mtx);
 
 	for (i = 0; i < IDX_COUNT; i++)
-		UNTIMEOUT((cps[i])->TO, (void *)sp, sp->ch[i]);
-	UNTIMEOUT(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
+		untimeout((cps[i])->TO, (void *)sp, sp->ch[i]);
+	untimeout(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
 	mtx_destroy(&sp->pp_cpq.ifq_mtx);
 	mtx_destroy(&sp->pp_fastq.ifq_mtx);
 }
@@ -2004,8 +2008,8 @@
 		case STATE_STOPPING:
 			sppp_cp_send(sp, cp->proto, TERM_REQ,
 				     ++sp->pp_seq[cp->protoidx], 0, 0);
-			TIMEOUT(cp->TO, (void *)sp, sp->lcp.timeout,
-			    sp->ch[cp->protoidx]);
+			sp->ch[cp->protoidx] = timeout(cp->TO, (void *)sp,
+			    sp->lcp.timeout);
 			break;
 		case STATE_REQ_SENT:
 		case STATE_ACK_RCVD:
@@ -2015,8 +2019,8 @@
 			break;
 		case STATE_ACK_SENT:
 			(cp->scr)(sp);
-			TIMEOUT(cp->TO, (void *)sp, sp->lcp.timeout,
-			    sp->ch[cp->protoidx]);
+			sp->ch[cp->protoidx] = timeout(cp->TO, (void *)sp,
+			    sp->lcp.timeout);
 			break;
 		}
 
@@ -2032,7 +2036,7 @@
 {
 	sp->state[cp->protoidx] = newstate;
 
-	UNTIMEOUT(cp->TO, (void *)sp, sp->ch[cp->protoidx]);
+	untimeout(cp->TO, (void *)sp, sp->ch[cp->protoidx]);
 	switch (newstate) {
 	case STATE_INITIAL:
 	case STATE_STARTING:
@@ -2045,8 +2049,8 @@
 	case STATE_REQ_SENT:
 	case STATE_ACK_RCVD:
 	case STATE_ACK_SENT:
-		TIMEOUT(cp->TO, (void *)sp, sp->lcp.timeout,
-		    sp->ch[cp->protoidx]);
+		sp->ch[cp->protoidx] = timeout(cp->TO, (void *)sp,
+		    sp->lcp.timeout);
 		break;
 	}
 }
@@ -4142,7 +4146,7 @@
 		 * a number between 300 and 810 seconds.
 		 */
 		i = 300 + ((unsigned)(random() & 0xff00) >> 7);
-		TIMEOUT(chap.TO, (void *)sp, i * hz, sp->ch[IDX_CHAP]);
+		sp->ch[IDX_CHAP] = timeout(chap.TO, (void *)sp, i * hz);
 	}
 
 	if (debug) {
@@ -4186,7 +4190,7 @@
 
 	if (debug)
 		log(LOG_DEBUG, SPP_FMT "chap tld\n", SPP_ARGS(ifp));
-	UNTIMEOUT(chap.TO, (void *)sp, sp->ch[IDX_CHAP]);
+	untimeout(chap.TO, (void *)sp, sp->ch[IDX_CHAP]);
 	sp->lcp.protos &= ~(1 << IDX_CHAP);
 
 	lcp.Close(sp);
@@ -4322,7 +4326,7 @@
 
 	/* ack and nak are his authproto */
 	case PAP_ACK:
-		UNTIMEOUT(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
+		untimeout(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
 		if (debug) {
 			log(LOG_DEBUG, SPP_FMT "pap success",
 			    SPP_ARGS(ifp));
@@ -4351,7 +4355,7 @@
 		break;
 
 	case PAP_NAK:
-		UNTIMEOUT(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
+		untimeout(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
 		if (debug) {
 			log(LOG_INFO, SPP_FMT "pap failure",
 			    SPP_ARGS(ifp));
@@ -4408,8 +4412,8 @@
 	if (sp->myauth.proto == PPP_PAP) {
 		/* we are peer, send a request, and start a timer */
 		pap.scr(sp);
-		TIMEOUT(sppp_pap_my_TO, (void *)sp, sp->lcp.timeout,
-		    sp->pap_my_to_ch);
+		sp->pap_my_to_ch = timeout(sppp_pap_my_TO, (void *)sp,
+		    sp->lcp.timeout);
 	}
 }
 
@@ -4512,8 +4516,8 @@
 
 	if (debug)
 		log(LOG_DEBUG, SPP_FMT "pap tld\n", SPP_ARGS(ifp));
-	UNTIMEOUT(pap.TO, (void *)sp, sp->ch[IDX_PAP]);
-	UNTIMEOUT(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
+	untimeout(pap.TO, (void *)sp, sp->ch[IDX_PAP]);
+	untimeout(sppp_pap_my_TO, (void *)sp, sp->pap_my_to_ch);
 	sp->lcp.protos &= ~(1 << IDX_PAP);
 
 	lcp.Close(sp);
@@ -4640,7 +4644,12 @@
 	struct sppp *sp;
 	int s;
 
+	/*
+	 * XXXRW: It would be nice to avoid calling all this stuff while
+	 * holding sppp_mtx, or we risk lock order reversals.
+	 */
 	s = splimp();
+	mtx_lock(&sppp_mtx);
 	for (sp=spppq; sp; sp=sp->pp_next) {
 		struct ifnet *ifp = &sp->pp_if;
 
@@ -4679,8 +4688,9 @@
 				sp->lcp.echoid, 4, &nmagic);
 		}
 	}
+	callout_reset(&keepalive_callout, hz * 10, sppp_keepalive, NULL);
+	mtx_unlock(&sppp_mtx);
 	splx(s);
-	TIMEOUT(sppp_keepalive, 0, hz * 10, keepalive_ch);
 }
 
 /*
--- //depot/vendor/freebsd/src/sys/net/if_stf.c	2004/03/09 12:30:36
+++ //depot/user/rwatson/netperf/sys/net/if_stf.c	2004/03/12 06:09:05
@@ -136,13 +136,11 @@
 #define sc_ro	__sc_ro46.__sc_ro4
 	const struct encaptab *encap_cookie;
 	LIST_ENTRY(stf_softc) sc_list;	/* all stf's are linked */
+	struct mtx	sc_mtx;		/* protect sc_ro */
 };
 
 /*
  * All mutable global variables in if_stf.c are protected by stf_mtx.
- * XXXRW: Note that mutable fields in the softc are not currently locked:
- * in particular, sc_ro needs to be protected from concurrent entrance
- * of stf_output().
  */
 static struct mtx stf_mtx;
 static LIST_HEAD(, stf_softc) stf_softc_list;
@@ -196,6 +194,7 @@
 		free(sc, M_STF);
 		return (ENOMEM);
 	}
+	mtx_init(&sc->sc_mtx, "stf sc_mtx", NULL, MTX_DEF);
 
 	sc->sc_if.if_mtu    = IPV6_MMTU;
 	sc->sc_if.if_ioctl  = stf_ioctl;
@@ -220,6 +219,7 @@
 	bpfdetach(&sc->sc_if);
 	if_detach(&sc->sc_if);
 
+	mtx_destroy(&sc->sc_mtx);
 	free(sc, M_STF);
 }
 
@@ -391,9 +391,10 @@
 	struct ip *ip;
 	struct ip6_hdr *ip6;
 	struct in6_ifaddr *ia6;
-#ifdef MAC
+	struct route ro;
 	int error;
 
+#ifdef MAC
 	error = mac_check_ifnet_transmit(ifp, m);
 	if (error) {
 		m_freem(m);
@@ -495,9 +496,7 @@
 	else
 		ip_ecn_ingress(ECN_NOCARE, &ip->ip_tos, &tos);
 
-	/*
-	 * XXXRW: Locking of sc_ro required.
-	 */
+	mtx_lock(&sc->sc_mtx);
 	dst4 = (struct sockaddr_in *)&sc->sc_ro.ro_dst;
 	if (dst4->sin_family != AF_INET ||
 	    bcmp(&dst4->sin_addr, &ip->ip_dst, sizeof(ip->ip_dst)) != 0) {
@@ -516,12 +515,21 @@
 		if (sc->sc_ro.ro_rt == NULL) {
 			m_freem(m);
 			ifp->if_oerrors++;
+			mtx_unlock(&sc->sc_mtx);
 			return ENETUNREACH;
 		}
 	}
 
+	/*
+	 * XXXRW: Holding mutex over call to ip_output(): potential lock
+	 * order issue?  Hard to resolve cleanly with the current route
+	 * caching model, as we have to synchronize access to shared softc
+	 * state.
+	 */
 	ifp->if_opackets++;
-	return ip_output(m, NULL, &sc->sc_ro, 0, NULL, NULL);
+	error = ip_output(m, NULL, &ro, 0, NULL, NULL);
+	mtx_unlock(&sc->sc_mtx);
+	return (error);
 }
 
 static int
--- //depot/vendor/freebsd/src/sys/net/if_tap.c	2004/03/11 05:02:01
+++ //depot/user/rwatson/netperf/sys/net/if_tap.c	2004/03/12 21:39:24
@@ -108,6 +108,12 @@
 	.d_name =	CDEV_NAME,
 };
 
+/*
+ * All global variables in if_tap.c are locked with tapmtx, with the
+ * exception of tapdebug, which is accessed unlocked; tapclones is
+ * static at runtime.
+ */
+static struct mtx		tapmtx;
 static int			tapdebug = 0;        /* debug flag   */
 static SLIST_HEAD(, tap_softc)	taphead;             /* first device */
 static struct clonedevs 	*tapclones;
@@ -138,28 +144,47 @@
 
 		/* intitialize device */
 
+		mtx_init(&tapmtx, "tapmtx", NULL, MTX_DEF);
 		SLIST_INIT(&taphead);
 
 		clone_setup(&tapclones);
 		eh_tag = EVENTHANDLER_REGISTER(dev_clone, tapclone, 0, 1000);
-		if (eh_tag == NULL)
+		if (eh_tag == NULL) {
+			mtx_destroy(&tapmtx);
 			return (ENOMEM);
+		}
 		return (0);
 
 	case MOD_UNLOAD:
-		SLIST_FOREACH(tp, &taphead, tap_next)
-			if (tp->tap_flags & TAP_OPEN)
+		/*
+		 * The EBUSY algorithm here can't quite atomically
+		 * guarantee that this is race-free since we have to
+		 * release the tap mtx to deregister the clone handler.
+		 */
+		mtx_lock(&tapmtx);
+		SLIST_FOREACH(tp, &taphead, tap_next) {
+			mtx_lock(&tp->tap_mtx);
+			if (tp->tap_flags & TAP_OPEN) {
+				mtx_unlock(&tp->tap_mtx);
+				mtx_unlock(&tapmtx);
 				return (EBUSY);
+			}
+			mtx_unlock(&tp->tap_mtx);
+		}
+		mtx_unlock(&tapmtx);
 
 		EVENTHANDLER_DEREGISTER(dev_clone, eh_tag);
 
+		mtx_lock(&tapmtx);
 		while ((tp = SLIST_FIRST(&taphead)) != NULL) {
 			SLIST_REMOVE_HEAD(&taphead, tap_next);
+			mtx_unlock(&tapmtx);
 
 			ifp = &tp->tap_if;
 
 			TAPDEBUG("detaching %s\n", ifp->if_xname);
 
+			/* Unlocked read. */
 			KASSERT(!(tp->tap_flags & TAP_OPEN), 
 				("%s flags is out of sync", ifp->if_xname));
 
@@ -168,8 +193,11 @@
 			ether_ifdetach(ifp);
 			splx(s);
 
+			mtx_destroy(&tp->tap_mtx);
 			free(tp, M_TAP);
+			mtx_lock(&tapmtx);
 		}
+		mtx_unlock(&tapmtx);
 		clone_cleanup(&tapclones);
 
 		break;
@@ -246,7 +274,10 @@
 
 	/* allocate driver storage and create device */
 	MALLOC(tp, struct tap_softc *, sizeof(*tp), M_TAP, M_WAITOK | M_ZERO);
+	mtx_init(&tp->tap_mtx, "tap_mtx", NULL, MTX_DEF);
+	mtx_lock(&tapmtx);
 	SLIST_INSERT_HEAD(&taphead, tp, tap_next);
+	mtx_unlock(&tapmtx);
 
 	unit = dev2unit(dev);
 
@@ -285,7 +316,9 @@
 	ether_ifattach(ifp, tp->arpcom.ac_enaddr);
 	splx(s);
 
+	mtx_lock(&tp->tap_mtx);
 	tp->tap_flags |= TAP_INITED;
+	mtx_unlock(&tp->tap_mtx);
 
 	TAPDEBUG("interface %s is created. minor = %#x\n", 
 		ifp->if_xname, minor(dev));
@@ -319,13 +352,16 @@
 		tp = dev->si_drv1;
 	}
 
+	/* Unlocked read. */
 	KASSERT(!(tp->tap_flags & TAP_OPEN), 
 		("%s flags is out of sync", tp->tap_if.if_xname));
 
 	bcopy(tp->arpcom.ac_enaddr, tp->ether_addr, sizeof(tp->ether_addr));
 
+	mtx_lock(&tp->tap_mtx);
 	tp->tap_pid = td->td_proc->p_pid;
 	tp->tap_flags |= TAP_OPEN;
+	mtx_unlock(&tp->tap_mtx);
 
 	TAPDEBUG("%s is open. minor = %#x\n", 
 		tp->tap_if.if_xname, minor(dev));
@@ -358,7 +394,9 @@
 	 * interface, if we are in VMnet mode. just close the device.
 	 */
 
+	mtx_lock(&tp->tap_mtx);
 	if (((tp->tap_flags & TAP_VMNET) == 0) && (ifp->if_flags & IFF_UP)) {
+		mtx_unlock(&tp->tap_mtx);
 		s = splimp();
 		if_down(ifp);
 		if (ifp->if_flags & IFF_RUNNING) {
@@ -382,13 +420,16 @@
 			ifp->if_flags &= ~IFF_RUNNING;
 		}
 		splx(s);
-	}
+	} else
+		mtx_unlock(&tp->tap_mtx);
 
 	funsetown(&tp->tap_sigio);
 	selwakeuppri(&tp->tap_rsel, PZERO+1);
 
+	mtx_lock(&tp->tap_mtx);
 	tp->tap_flags &= ~TAP_OPEN;
 	tp->tap_pid = 0;
+	mtx_unlock(&tp->tap_mtx);
 
 	TAPDEBUG("%s is closed. minor = %#x\n", 
 		ifp->if_xname, minor(dev));
@@ -444,10 +485,12 @@
 			s = splimp();
 			ifs = (struct ifstat *)data;
 			dummy = strlen(ifs->ascii);
+			mtx_lock(&tp->tap_mtx);
 			if (tp->tap_pid != 0 && dummy < sizeof(ifs->ascii))
 				snprintf(ifs->ascii + dummy,
 					sizeof(ifs->ascii) - dummy,
 					"\tOpened by PID %d\n", tp->tap_pid);
+			mtx_unlock(&tp->tap_mtx);
 			splx(s);
 			break;
 
@@ -481,10 +524,14 @@
 	 * XXX: can this do any harm because of queue overflow?
 	 */
 
+	mtx_lock(&tp->tap_mtx);
 	if (((tp->tap_flags & TAP_VMNET) == 0) &&
 	    ((tp->tap_flags & TAP_READY) != TAP_READY)) {
 		struct mbuf	*m = NULL;
 
+		mtx_unlock(&tp->tap_mtx);
+
+		/* Unlocked read. */
 		TAPDEBUG("%s not ready, tap_flags = 0x%x\n", ifp->if_xname, 
 		    tp->tap_flags);
 
@@ -504,13 +551,17 @@
 	ifp->if_flags |= IFF_OACTIVE;
 
 	if (ifp->if_snd.ifq_len != 0) {
+		mtx_lock(&tp->tap_mtx);
 		if (tp->tap_flags & TAP_RWAIT) {
 			tp->tap_flags &= ~TAP_RWAIT;
 			wakeup(tp);
 		}
 
-		if ((tp->tap_flags & TAP_ASYNC) && (tp->tap_sigio != NULL))
+		if ((tp->tap_flags & TAP_ASYNC) && (tp->tap_sigio != NULL)) {
+			mtx_unlock(&tp->tap_mtx);
 			pgsigio(&tp->tap_sigio, SIGIO, 0);
+		} else
+			mtx_unlock(&tp->tap_mtx);
 
 		selwakeuppri(&tp->tap_rsel, PZERO+1);
 		ifp->if_opackets ++; /* obytes are counted in ether_output */
@@ -570,10 +621,12 @@
 
 		case FIOASYNC:
 			s = splimp();
+			mtx_lock(&tp->tap_mtx);
 			if (*(int *)data)
 				tp->tap_flags |= TAP_ASYNC;
 			else
 				tp->tap_flags &= ~TAP_ASYNC;
+			mtx_unlock(&tp->tap_mtx);
 			splx(s);
 			break;
 
@@ -628,6 +681,7 @@
 			break;
 
 		case SIOCSIFADDR:	/* set MAC address of the remote side */
+			/* XXXRW: Does this actually do anything? */
 			bcopy(data, tp->ether_addr, sizeof(tp->ether_addr));
 			break;
 
@@ -657,7 +711,11 @@
 
 	TAPDEBUG("%s reading, minor = %#x\n", ifp->if_xname, minor(dev));
 
+	mtx_lock(&tp->tap_mtx);
 	if ((tp->tap_flags & TAP_READY) != TAP_READY) {
+		mtx_unlock(&tp->tap_mtx);
+
+		/* Unlocked read. */
 		TAPDEBUG("%s not ready. minor = %#x, tap_flags = 0x%x\n",
 			ifp->if_xname, minor(dev), tp->tap_flags);
 
@@ -665,6 +723,7 @@
 	}
 
 	tp->tap_flags &= ~TAP_RWAIT;
+	mtx_unlock(&tp->tap_mtx);
 
 	/* sleep until we get a packet */
 	do {
@@ -676,7 +735,10 @@
 			if (flag & IO_NDELAY)
 				return (EWOULDBLOCK);
 
+			/* This looks like a wanna-be condition variable. */
+			mtx_lock(&tp->tap_mtx);
 			tp->tap_flags |= TAP_RWAIT;
+			mtx_unlock(&tp->tap_mtx);
 			error = tsleep(tp,PCATCH|(PZERO+1),"taprd",0);
 			if (error)
 				return (error);
--- //depot/vendor/freebsd/src/sys/net/if_tapvar.h	2004/03/12 21:51:47
+++ //depot/user/rwatson/netperf/sys/net/if_tapvar.h	2004/03/12 22:29:12
@@ -41,6 +41,10 @@
 #ifndef _NET_IF_TAPVAR_H_
 #define _NET_IF_TAPVAR_H_
 
+/*
+ * tap_mtx locks tap_flags, tap_pid.  tap_next locked with global tapmtx.
+ * Other fields locked by owning subsystems.
+ */
 struct tap_softc {
 	struct arpcom	arpcom;			/* ethernet common data      */
 #define tap_if		arpcom.ac_if
@@ -60,6 +64,7 @@
 
 	SLIST_ENTRY(tap_softc)	tap_next;	/* next device in chain      */
 	dev_t		 tap_dev;
+	struct mtx	 tap_mtx;		/* per-softc mutex */
 };
 
 #endif /* !_NET_IF_TAPVAR_H_ */
--- //depot/vendor/freebsd/src/sys/net/if_tun.c	2004/03/12 21:51:47
+++ //depot/user/rwatson/netperf/sys/net/if_tun.c	2004/03/12 22:29:12
@@ -55,6 +55,10 @@
 
 #include <sys/queue.h>
 
+/*
+ * tun_list is protected by global tunmtx.  Other mutable fields are
+ * protected by tun->tun_mtx, or by their owning subsystm.
+ */
 struct tun_softc {
 	TAILQ_ENTRY(tun_softc)	tun_list;
 	dev_t			tun_dev;
@@ -75,11 +79,18 @@
 	struct	ifnet tun_if;		/* the interface */
 	struct  sigio *tun_sigio;	/* information for async I/O */
 	struct	selinfo	tun_rsel;	/* read select */
+	struct mtx	tun_mtx;	/* protect mutable softc fields */
 };
 
 #define TUNDEBUG	if (tundebug) if_printf
 #define	TUNNAME		"tun"
 
+/*
+ * All mutable global variables in if_tun are locked using tunmtx, with
+ * the exception of tundebug, which is used unlocked, and tunclones,
+ * which is static after setup.
+ */
+static struct mtx tunmtx;
 static MALLOC_DEFINE(M_TUN, TUNNAME, "Tunnel Interface");
 static int tundebug = 0;
 static struct clonedevs *tunclones;
@@ -149,6 +160,7 @@
 
 	switch (type) {
 	case MOD_LOAD:
+		mtx_init(&tunmtx, "tunmtx", NULL, MTX_DEF);
 		clone_setup(&tunclones);
 		tag = EVENTHANDLER_REGISTER(dev_clone, tunclone, 0, 1000);
 		if (tag == NULL)
@@ -157,19 +169,25 @@
 	case MOD_UNLOAD:
 		EVENTHANDLER_DEREGISTER(dev_clone, tag);
 
-		while (!TAILQ_EMPTY(&tunhead)) {
-			tp = TAILQ_FIRST(&tunhead);
+		mtx_lock(&tunmtx);
+		while ((tp = TAILQ_FIRST(&tunhead)) != NULL) {
+			TAILQ_REMOVE(&tunhead, tp, tun_list);
+			mtx_unlock(&tunmtx);
+			/* Unlocked read. */
 			KASSERT((tp->tun_flags & TUN_OPEN) == 0,
 			    ("tununits is out of sync - unit %d",
 			    tp->tun_if.if_dunit));
-			TAILQ_REMOVE(&tunhead, tp, tun_list);
 			dev = tp->tun_dev;
 			bpfdetach(&tp->tun_if);
 			if_detach(&tp->tun_if);
 			destroy_dev(dev);
+			mtx_destroy(&tp->tun_mtx);
 			free(tp, M_TUN);
+			mtx_lock(&tunmtx);
 		}
+		mtx_unlock(&tunmtx);
 		clone_cleanup(&tunclones);
+		mtx_destroy(&tunmtx);
 		break;
 	}
 	return 0;
@@ -188,12 +206,16 @@
 {
 	struct tun_softc *tp = ifp->if_softc;
 
+	mtx_lock(&tp->tun_mtx);
 	if (tp->tun_flags & TUN_RWAIT) {
 		tp->tun_flags &= ~TUN_RWAIT;
 		wakeup(tp);
 	}
-	if (tp->tun_flags & TUN_ASYNC && tp->tun_sigio)
+	if (tp->tun_flags & TUN_ASYNC && tp->tun_sigio) {
+		mtx_unlock(&tp->tun_mtx);
 		pgsigio(&tp->tun_sigio, SIGIO, 0);
+	} else
+		mtx_unlock(&tp->tun_mtx);
 	selwakeuppri(&tp->tun_rsel, PZERO + 1);
 }
 
@@ -206,9 +228,12 @@
 	dev->si_flags &= ~SI_CHEAPCLONE;
 
 	MALLOC(sc, struct tun_softc *, sizeof(*sc), M_TUN, M_WAITOK | M_ZERO);
+	mtx_init(&sc->tun_mtx, "tun_mtx", NULL, MTX_DEF);
 	sc->tun_flags = TUN_INITED;
 	sc->tun_dev = dev;
+	mtx_lock(&tunmtx);
 	TAILQ_INSERT_TAIL(&tunhead, sc, tun_list);
+	mtx_unlock(&tunmtx);
 
 	ifp = &sc->tun_if;
 	if_initname(ifp, TUNNAME, dev2unit(dev));
@@ -237,11 +262,21 @@
 		tp = dev->si_drv1;
 	}
 
-	if (tp->tun_proc != NULL && tp->tun_proc != td->td_proc)
+	/*
+	 * XXXRW: Storing tun_proc seems inherrently unsafe, as the
+	 * tunnel device reference might live beyond the process itself.
+	 * Prhaps we should be caching a pid and/or using a generation
+	 * number.  Or, a busy flag -- does the if_tun interface need
+	 * to be opened more than once at a time?
+	 */
+	mtx_lock(&tp->tun_mtx);
+	if (tp->tun_proc != NULL && tp->tun_proc != td->td_proc) {
+		mtx_unlock(&tp->tun_mtx);
 		return (EBUSY);
+	}
 	tp->tun_proc = td->td_proc;
-
 	tp->tun_flags |= TUN_OPEN;
+	mtx_unlock(&tp->tun_mtx);
 	ifp = &tp->tun_if;
 	TUNDEBUG(ifp, "open\n");
 
@@ -262,8 +297,10 @@
 	tp = dev->si_drv1;
 	ifp = &tp->tun_if;
 
+	mtx_lock(&tp->tun_mtx);
 	tp->tun_flags &= ~TUN_OPEN;
 	tp->tun_proc = NULL;
+	mtx_unlock(&tp->tun_mtx);
 
 	/*
 	 * junk all pending output
@@ -283,6 +320,7 @@
 		/* find internet addresses and delete routes */
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
 			if (ifa->ifa_addr->sa_family == AF_INET)
+				/* Unlocked read. */
 				rtinit(ifa, (int)RTM_DELETE,
 				    tp->tun_flags & TUN_DSTADDR ? RTF_HOST : 0);
 		ifp->if_flags &= ~IFF_RUNNING;
@@ -307,6 +345,9 @@
 	ifp->if_flags |= IFF_UP | IFF_RUNNING;
 	getmicrotime(&ifp->if_lastchange);
 
+	/*
+	 * XXXRW: interface locking.
+	 */
 	for (ifa = TAILQ_FIRST(&ifp->if_addrhead); ifa;
 	     ifa = TAILQ_NEXT(ifa, ifa_link)) {
 		if (ifa->ifa_addr == NULL)
@@ -318,12 +359,14 @@
 			    struct sockaddr_in *si;
 
 			    si = (struct sockaddr_in *)ifa->ifa_addr;
+			    mtx_lock(&tp->tun_mtx);
 			    if (si->sin_addr.s_addr)
 				    tp->tun_flags |= TUN_IASET;
 
 			    si = (struct sockaddr_in *)ifa->ifa_dstaddr;
 			    if (si && si->sin_addr.s_addr)
 				    tp->tun_flags |= TUN_DSTADDR;
+			    mtx_unlock(&tp->tun_mtx);
 			}
 #endif
 		}
@@ -346,9 +389,11 @@
 	switch(cmd) {
 	case SIOCGIFSTATUS:
 		ifs = (struct ifstat *)data;
+		mtx_lock(&tp->tun_mtx);
 		if (tp->tun_proc)
 			sprintf(ifs->ascii + strlen(ifs->ascii),
 			    "\tOpened by PID %d\n", tp->tun_proc->p_pid);
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case SIOCSIFADDR:
 		error = tuninit(ifp);
@@ -384,6 +429,7 @@
 	struct rtentry *rt)
 {
 	struct tun_softc *tp = ifp->if_softc;
+	u_short cached_tun_flags;
 #ifdef MAC
 	int error;
 #endif
@@ -398,7 +444,11 @@
 	}
 #endif
 
-	if ((tp->tun_flags & TUN_READY) != TUN_READY) {
+	/* Could be unlocked read? */
+	mtx_lock(&tp->tun_mtx);
+	cached_tun_flags = tp->tun_flags;
+	mtx_unlock(&tp->tun_mtx);
+	if ((cached_tun_flags & TUN_READY) != TUN_READY) {
 		TUNDEBUG (ifp, "not ready 0%o\n", tp->tun_flags);
 		m_freem (m0);
 		return (EHOSTDOWN);
@@ -423,7 +473,7 @@
 	}
 
 	/* prepend sockaddr? this may abort if the mbuf allocation fails */
-	if (tp->tun_flags & TUN_LMODE) {
+	if (cached_tun_flags & TUN_LMODE) {
 		/* allocate space for sockaddr */
 		M_PREPEND(m0, dst->sa_len, M_DONTWAIT);
 
@@ -437,7 +487,7 @@
 		}
 	}
 
-	if (tp->tun_flags & TUN_IFHEAD) {
+	if (cached_tun_flags & TUN_IFHEAD) {
 		/* Prepend the address family */
 		M_PREPEND(m0, 4, M_DONTWAIT);
 
@@ -502,21 +552,28 @@
 		*(int *)data = tundebug;
 		break;
 	case TUNSLMODE:
+		mtx_lock(&tp->tun_mtx);
 		if (*(int *)data) {
 			tp->tun_flags |= TUN_LMODE;
 			tp->tun_flags &= ~TUN_IFHEAD;
 		} else
 			tp->tun_flags &= ~TUN_LMODE;
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case TUNSIFHEAD:
+		mtx_lock(&tp->tun_mtx);
 		if (*(int *)data) {
 			tp->tun_flags |= TUN_IFHEAD;
 			tp->tun_flags &= ~TUN_LMODE;
 		} else
 			tp->tun_flags &= ~TUN_IFHEAD;
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case TUNGIFHEAD:
+		/* Could be unlocked read? */
+		mtx_lock(&tp->tun_mtx);
 		*(int *)data = (tp->tun_flags & TUN_IFHEAD) ? 1 : 0;
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case TUNSIFMODE:
 		/* deny this if UP */
@@ -535,15 +592,22 @@
 		}
 		break;
 	case TUNSIFPID:
+		/*
+		 * XXXRW: This is scary too.
+		 */
+		mtx_lock(&tp->tun_mtx);
 		tp->tun_proc = curthread->td_proc;
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case FIONBIO:
 		break;
 	case FIOASYNC:
+		mtx_lock(&tp->tun_mtx);
 		if (*(int *)data)
 			tp->tun_flags |= TUN_ASYNC;
 		else
 			tp->tun_flags &= ~TUN_ASYNC;
+		mtx_unlock(&tp->tun_mtx);
 		break;
 	case FIONREAD:
 		s = splimp();
@@ -590,12 +654,15 @@
 	int		error=0, len, s;
 
 	TUNDEBUG (ifp, "read\n");
+	mtx_lock(&tp->tun_mtx);
 	if ((tp->tun_flags & TUN_READY) != TUN_READY) {
+		mtx_unlock(&tp->tun_mtx);
 		TUNDEBUG (ifp, "not ready 0%o\n", tp->tun_flags);
 		return (EHOSTDOWN);
 	}
 
 	tp->tun_flags &= ~TUN_RWAIT;
+	mtx_unlock(&tp->tun_mtx);
 
 	s = splimp();
 	do {
@@ -605,7 +672,9 @@
 				splx(s);
 				return (EWOULDBLOCK);
 			}
+			mtx_lock(&tp->tun_mtx);
 			tp->tun_flags |= TUN_RWAIT;
+			mtx_unlock(&tp->tun_mtx);
 			if((error = tsleep(tp, PCATCH | (PZERO + 1),
 					"tunread", 0)) != 0) {
 				splx(s);
@@ -692,14 +761,19 @@
 	mac_create_mbuf_from_ifnet(ifp, top);
 #endif
 
+	/* Could be unlocked read? */
+	mtx_lock(&tp->tun_mtx);
 	if (tp->tun_flags & TUN_IFHEAD) {
+		mtx_unlock(&tp->tun_mtx);
 		if (top->m_len < sizeof(family) &&
 		    (top = m_pullup(top, sizeof(family))) == NULL)
 			return (ENOBUFS);
 		family = ntohl(*mtod(top, u_int32_t *));
 		m_adj(top, sizeof(family));
-	} else
+	} else {
+		mtx_unlock(&tp->tun_mtx);
 		family = AF_INET;
+	}
 
 	BPF_MTAP2(ifp, &family, sizeof(family), top);
 
--- //depot/vendor/freebsd/src/sys/net/raw_cb.c	2003/02/18 21:53:12
+++ //depot/user/rwatson/netperf/sys/net/raw_cb.c	2004/03/11 18:54:06
@@ -36,7 +36,9 @@
 
 #include <sys/param.h>
 #include <sys/domain.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/protosw.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
@@ -53,10 +55,11 @@
  *	redo address binding to allow wildcards
  */
 
+struct mtx rawcb_mtx;
 struct rawcb_list_head rawcb_list;
 
-static u_long	raw_sendspace = RAWSNDQ;
-static u_long	raw_recvspace = RAWRCVQ;
+static const u_long	raw_sendspace = RAWSNDQ;
+static const u_long	raw_recvspace = RAWRCVQ;
 
 /*
  * Allocate a control block and a nominal amount
@@ -83,7 +86,9 @@
 	rp->rcb_socket = so;
 	rp->rcb_proto.sp_family = so->so_proto->pr_domain->dom_family;
 	rp->rcb_proto.sp_protocol = proto;
+	mtx_lock(&rawcb_mtx);
 	LIST_INSERT_HEAD(&rawcb_list, rp, list);
+	mtx_unlock(&rawcb_mtx);
 	return (0);
 }
 
@@ -97,6 +102,7 @@
 {
 	struct socket *so = rp->rcb_socket;
 
+	SOCK_LOCK(so);
 	so->so_pcb = 0;
 	sotryfree(so);
 	LIST_REMOVE(rp, list);
--- //depot/vendor/freebsd/src/sys/net/raw_cb.h	2002/03/19 13:55:39
+++ //depot/user/rwatson/netperf/sys/net/raw_cb.h	2004/03/11 18:54:06
@@ -61,6 +61,7 @@
 
 #ifdef _KERNEL
 extern LIST_HEAD(rawcb_list_head, rawcb) rawcb_list;
+extern struct mtx rawcb_mtx;
 
 /* protosw entries */
 pr_ctlinput_t	raw_ctlinput;
--- //depot/vendor/freebsd/src/sys/net/raw_usrreq.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/net/raw_usrreq.c	2004/03/11 18:54:06
@@ -38,6 +38,8 @@
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/protosw.h>
 #include <sys/signalvar.h>
 #include <sys/socket.h>
@@ -53,6 +55,8 @@
 void
 raw_init()
 {
+
+	mtx_init(&rawcb_mtx, "rawcb_mtx", NULL, MTX_DEF);
 	LIST_INIT(&rawcb_list);
 }
 
@@ -75,7 +79,12 @@
 	register struct mbuf *m = m0;
 	struct socket *last;
 
+	/*
+	 * XXXRW: Potential lock order issues due to holding the
+	 * rawcb_mtx across all this stuff.  Need to revisit.
+	 */
 	last = 0;
+	mtx_lock(&rawcb_mtx);
 	LIST_FOREACH(rp, &rawcb_list, list) {
 		if (rp->rcb_proto.sp_family != proto->sp_family)
 			continue;
@@ -120,6 +129,7 @@
 		}
 	} else
 		m_freem(m);
+	mtx_unlock(&rawcb_mtx);
 }
 
 /*ARGSUSED*/
@@ -143,8 +153,12 @@
 	if (rp == 0)
 		return EINVAL;
 	raw_disconnect(rp);
-	sotryfree(so);
-	soisdisconnected(so);	/* XXX huh? called after the sofree()? */
+	SOCK_LOCK(so);
+	if (so->so_count != 0) {
+		soisdisconnected(so);
+		SOCK_UNLOCK(so);
+	} else
+		sofree(so);
 	return 0;
 }
 
--- //depot/vendor/freebsd/src/sys/net/rtsock.c	2003/11/20 12:10:44
+++ //depot/user/rwatson/netperf/sys/net/rtsock.c	2004/03/11 18:56:12
@@ -55,10 +55,18 @@
 MALLOC_DEFINE(M_RTABLE, "routetbl", "routing tables");
 
 /* NB: these are not modified */
+/*
+ * XXXRW: It would be really nice to add const to these, but that may
+ * not be possible due to where they are passed in.  We might need
+ * to const-poison a whole boatload of APIs...?
+ */
 static struct	sockaddr route_dst = { 2, PF_ROUTE, };
 static struct	sockaddr route_src = { 2, PF_ROUTE, };
 static struct	sockaddr sa_zero   = { sizeof(sa_zero), AF_INET, };
 
+/*
+ * XXXRW: These fields are locked by RTSOCK_LOCK().
+ */
 static struct {
 	int	ip_count;	/* attacked w/ AF_INET */
 	int	ip6_count;	/* attached w/ AF_INET6 */
--- //depot/vendor/freebsd/src/sys/netatalk/aarp.c	2004/03/12 21:30:33
+++ //depot/user/rwatson/netperf/sys/netatalk/aarp.c	2004/03/12 22:29:12
@@ -36,11 +36,15 @@
 #define AARPTAB_SIZE	(AARPTAB_BSIZ * AARPTAB_NB)
 static struct aarptab	aarptab[AARPTAB_SIZE];
 
+static struct mtx aarptab_mtx;
+MTX_SYSINIT(aarptab_mtx, &aarptab_mtx, "aarptab_mtx", MTX_DEF);
+
 #define AARPTAB_HASH(a) \
     ((((a).s_net << 8 ) + (a).s_node ) % AARPTAB_NB )
 
 #define AARPTAB_LOOK(aat,addr) { \
     int		n; \
+    mtx_assert(&aarptab_mtx, MA_OWNED); \
     aat = &aarptab[ AARPTAB_HASH(addr) * AARPTAB_BSIZ ]; \
     for ( n = 0; n < AARPTAB_BSIZ; n++, aat++ ) \
 	if ( aat->aat_ataddr.s_net == (addr).s_net && \
@@ -54,6 +58,9 @@
 #define AARPT_KILLC	20
 #define AARPT_KILLI	3
 
+/*
+ * XXXRW: wot?
+ */
 # if !defined( __FreeBSD__ )
 extern u_char			etherbroadcastaddr[6];
 # endif /* __FreeBSD__ */
@@ -63,7 +70,7 @@
 };
 
 /*
- * Not used?
+ * XXXRW: unused?
  */
 u_char	at_org_code[ 3 ] = {
     0x08, 0x00, 0x07,
@@ -72,6 +79,9 @@
     0x00, 0x00, 0x00,
 };
 
+/*
+ * XXXRW: Make use callouts, not timeouts.
+ */
 static struct callout_handle aarptimer_ch =
     CALLOUT_HANDLE_INITIALIZER(&aarptimer_ch);
 
@@ -83,6 +93,7 @@
 
     aarptimer_ch = timeout( aarptimer, (caddr_t)0, AARPT_AGE * hz );
     aat = aarptab;
+    mtx_lock(&aarptab_mtx);
     for ( i = 0; i < AARPTAB_SIZE; i++, aat++ ) {
 	if ( aat->aat_flags == 0 || ( aat->aat_flags & ATF_PERM ))
 	    continue;
@@ -93,6 +104,7 @@
 	aarptfree( aat );
 	splx( s );
     }
+    mtx_unlock(&aarptab_mtx);
 }
 
 /* 
@@ -130,6 +142,7 @@
     struct llc		*llc;
     struct sockaddr	sa;
 
+    mtx_assert(&aarptab_mtx, MA_NOTOWNED);
     if (( m = m_gethdr( M_DONTWAIT, MT_DATA )) == NULL ) {
 	return;
     }
@@ -232,6 +245,7 @@
     }
 
     s = splimp();
+    mtx_lock(&aarptab_mtx); /* XXXRW */
     AARPTAB_LOOK( aat, destsat->sat_addr );
     if ( aat == 0 ) {			/* No entry */
 	aat = aarptnew( &destsat->sat_addr );
@@ -239,6 +253,7 @@
 	    panic( "aarpresolve: no free entry" );
 	}
 	aat->aat_hold = m;
+	mtx_unlock(&aarptab_mtx);
 	aarpwhohas( ac, destsat );
 	splx( s );
 	return( 0 );
@@ -248,6 +263,7 @@
     if ( aat->aat_flags & ATF_COM ) {	/* entry is COMplete */
 	bcopy( (caddr_t)aat->aat_enaddr, (caddr_t)desten,
 		sizeof( aat->aat_enaddr ));
+	mtx_unlock(&aarptab_mtx);
 	splx( s );
 	return( 1 );
     }
@@ -256,6 +272,7 @@
 	m_freem( aat->aat_hold );
     }
     aat->aat_hold = m;
+    mtx_unlock(&aarptab_mtx);
     aarpwhohas( ac, destsat );
     splx( s );
     return( 0 );
@@ -390,6 +407,7 @@
 	}
     }
 
+    mtx_lock(&aarptab_mtx); /* XXXRW */
     AARPTAB_LOOK( aat, spa );
     if ( aat ) {
 	if ( op == AARPOP_PROBE ) {
@@ -399,6 +417,7 @@
 	     * to arp for him.
 	     */
 	    aarptfree( aat );
+	    mtx_unlock(&aarptab_mtx);
 	    m_freem( m );
 	    return;
 	}
@@ -409,12 +428,14 @@
 	if ( aat->aat_hold ) {
 	    struct mbuf *mhold = aat->aat_hold;
 	    aat->aat_hold = NULL;
+	    mtx_unlock(&aarptab_mtx);
 	    sat.sat_len = sizeof(struct sockaddr_at);
 	    sat.sat_family = AF_APPLETALK;
 	    sat.sat_addr = spa;
 	    (*ac->ac_if.if_output)( &ac->ac_if, mhold,
 		    (struct sockaddr *)&sat, NULL); /* XXX */
-	}
+	} else
+	    mtx_unlock(&aarptab_mtx);
     } else if ((tpa.s_net == ma.s_net)
 	   && (tpa.s_node == ma.s_node)
 	   && (op != AARPOP_PROBE)
@@ -422,7 +443,9 @@
 	        bcopy(( caddr_t )ea->aarp_sha, ( caddr_t )aat->aat_enaddr,
 		    sizeof( ea->aarp_sha ));
 	        aat->aat_flags |= ATF_COM;
-    }
+	        mtx_unlock(&aarptab_mtx);
+    } else
+	mtx_unlock(&aarptab_mtx);
 
     /*
      * Don't respond to responses, and never respond if we're
@@ -477,6 +500,7 @@
 aarptfree( struct aarptab *aat)
 {
 
+    mtx_assert(&aarptab_mtx, MTX_DEF);
     if ( aat->aat_hold )
 	m_freem( aat->aat_hold );
     aat->aat_hold = NULL;
@@ -485,7 +509,7 @@
     aat->aat_ataddr.s_node = 0;
 }
 
-    struct aarptab *
+struct aarptab *
 aarptnew( addr )
     struct at_addr	*addr;
 {
@@ -494,6 +518,7 @@
     struct aarptab	*aat, *aato = NULL;
     static int		first = 1;
 
+    mtx_assert(&aarptab_mtx, MA_OWNED);
     if ( first ) {
 	first = 0;
 	aarptimer_ch = timeout( aarptimer, (caddr_t)0, hz );
@@ -625,11 +650,16 @@
     struct aarptab	*aat;
     int			i;
 
+    /*
+     * XXXRW: Should grab mutex before untimeout?
+     */
     untimeout( aarptimer, 0, aarptimer_ch );
+    mtx_lock(&aarptab_mtx);
     for ( i = 0, aat = aarptab; i < AARPTAB_SIZE; i++, aat++ ) {
 	if ( aat->aat_hold ) {
 	    m_freem( aat->aat_hold );
 	    aat->aat_hold = NULL;
 	}
     }
+    mtx_unlock(&aarptab_mtx);
 }
--- //depot/vendor/freebsd/src/sys/netatalk/at_rmx.c	2001/07/17 00:20:42
+++ //depot/user/rwatson/netperf/sys/netatalk/at_rmx.c	2004/03/11 18:45:48
@@ -39,11 +39,17 @@
 
 int at_inithead(void **head, int off);
 
-static char hexbuf[256];
+/*
+ * XXXRW: hexdump was a static global variable, but I moved it into the
+ * stack rather than stick a mutex around it.  256 bytes is smaller than
+ * it used to be, but this still might be a problem.  Needs to be
+ * revisited.
+ */
 
 static char *
 prsockaddr(void *v)
 {
+	static char hexbuf[256];
 	char *bp = &hexbuf[0];
 	u_char *cp = v;
 
--- //depot/vendor/freebsd/src/sys/netatalk/ddp_input.c	2003/11/08 14:30:39
+++ //depot/user/rwatson/netperf/sys/netatalk/ddp_input.c	2004/03/11 18:46:40
@@ -29,6 +29,12 @@
 static volatile int	ddp_forward = 1;
 static volatile int	ddp_firewall = 0;
 static struct ddpstat	ddpstat;
+
+/*
+ * XXXRW: If we're going to keep this cached route data, we'll need to lock it
+ * down, and change later function-local use of it to grab an extra reference
+ * after deciding it is useful.
+ */
 static struct route	forwro;
 
 static void     ddp_input(struct mbuf *, struct ifnet *, struct elaphdr *, int);
--- //depot/vendor/freebsd/src/sys/netatalk/ddp_usrreq.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/netatalk/ddp_usrreq.c	2004/03/11 18:47:12
@@ -30,10 +30,19 @@
 static void at_pcbdetach(struct socket *so, struct ddpcb *ddp);
 static int at_pcballoc(struct socket *so);
 
+/*
+ * XXXRW: These structures require locking.
+ */
 struct ddpcb	*ddp_ports[ ATPORT_LAST ];
 struct ddpcb	*ddpcb = NULL;
-static u_long	ddp_sendspace = DDP_MAXSZ; /* Max ddp size + 1 (ddp_type) */
-static u_long	ddp_recvspace = 10 * ( 587 + sizeof( struct sockaddr_at ));
+
+/*
+ * XXXRW: These structures are currently not mutable.
+ */
+static const u_long	ddp_sendspace = DDP_MAXSZ; /* Max ddp size + 1
+			    (ddp_type) */
+static const u_long	ddp_recvspace = 10 * ( 587 + sizeof( struct
+			    sockaddr_at ));
 
 static struct ifqueue atintrq1, atintrq2, aarpintrq;
 
@@ -441,6 +450,7 @@
 static void
 at_pcbdetach( struct socket *so, struct ddpcb *ddp)
 {
+    SOCK_LOCK(so);
     soisdisconnected( so );
     so->so_pcb = 0;
     sotryfree(so);
--- //depot/vendor/freebsd/src/sys/netatm/atm_socket.c	2003/10/31 10:36:05
+++ //depot/user/rwatson/netperf/sys/netatm/atm_socket.c	2004/02/28 14:29:37
@@ -173,6 +173,7 @@
 	/*
 	 * Break links and free control blocks
 	 */
+	SOCK_LOCK(so);
 	so->so_pcb = NULL;
 	sotryfree(so);
 
--- //depot/vendor/freebsd/src/sys/netinet/if_ether.c	2003/12/23 06:10:35
+++ //depot/user/rwatson/netperf/sys/netinet/if_ether.c	2004/03/11 19:02:57
@@ -102,6 +102,11 @@
 #define la_timer la_rt->rt_rmx.rmx_expire /* deletion time in seconds */
 };
 
+/*
+ * XXXRW: Need to document (and/or fix) locking for this.  We always
+ * seem to hold a lock (and assert) when referencing this list, but it's
+ * not clear it's always the same lock.
+ */
 static	LIST_HEAD(, llinfo_arp) llinfo_arp;
 
 static struct	ifqueue arpintrq;
--- //depot/vendor/freebsd/src/sys/netinet/igmp.c	2003/08/28 15:15:24
+++ //depot/user/rwatson/netperf/sys/netinet/igmp.c	2004/03/11 19:06:01
@@ -84,10 +84,28 @@
 SYSCTL_STRUCT(_net_inet_igmp, IGMPCTL_STATS, stats, CTLFLAG_RW, &igmpstat,
     igmpstat, "");
 
+/*
+ * igmp_mtx protects all mutable global variables in igmp.c, as well as
+ * the data fields in struct router_info.  In general, a router_info
+ * structure will be valid as long as the referencing struct in_multi is
+ * valid, so no reference counting is used.  We allow unlocked reads of
+ * router_info data when accessed via an in_multi read-only.
+ */
+static struct mtx igmp_mtx;
 static SLIST_HEAD(, router_info) router_info_head;
 static int igmp_timers_are_running;
+
+/*
+ * XXXRW: can we define these such that these can be made const?  In any
+ * case, these shouldn't be changed after igmp_init() and therefore don't
+ * need locking.
+ */
 static u_long igmp_all_hosts_group;
 static u_long igmp_all_rtrs_group;
+
+/*
+ * XXXRW: These variables make me vaguely nervous.
+ */
 static struct mbuf *router_alert;
 static struct route igmprt;
 
@@ -112,6 +130,7 @@
 
 	/*
 	 * Construct a Router Alert option to use in outgoing packets
+	 * XXXRW: This might actually need a MAC label.
 	 */
 	MGET(router_alert, M_DONTWAIT, MT_DATA);
 	ra = mtod(router_alert, struct ipoption *);
@@ -122,6 +141,7 @@
 	ra->ipopt_list[3] = 0x00;
 	router_alert->m_len = sizeof(ra->ipopt_dst) + ra->ipopt_list[1];
 
+	mtx_init(&igmp_mtx, "igmp_mtx", NULL, MTX_DEF);
 	SLIST_INIT(&router_info_head);
 }
 
@@ -130,6 +150,7 @@
 {
 	struct router_info *rti;
 
+	mtx_assert(&igmp_mtx, MA_OWNED);
 	IGMP_PRINTF("[igmp.c, _find_rti] --> entering \n");
 	SLIST_FOREACH(rti, &router_info_head, rti_list) {
 		if (rti->rti_ifp == ifp) {
@@ -138,6 +159,9 @@
 			return rti;
 		}
 	}
+	/*
+	 * XXXRW: return value of malloc not checked, despite M_NOWAIT.
+	 */
 	MALLOC(rti, struct router_info *, sizeof *rti, M_IGMP, M_NOWAIT);
 	rti->rti_ifp = ifp;
 	rti->rti_type = IGMP_V2_ROUTER;
@@ -201,7 +225,6 @@
 	timer = igmp->igmp_code * PR_FASTHZ / IGMP_TIMER_SCALE;
 	if (timer == 0)
 		timer = 1;
-	rti = find_rti(ifp);
 
 	/*
 	 * In the IGMPv2 specification, there are 3 states and a flag.
@@ -228,8 +251,11 @@
 			 * value in RFC 1112.
 			 */
 
+			mtx_lock(&igmp_mtx);
+			rti = find_rti(ifp);
 			rti->rti_type = IGMP_V1_ROUTER;
 			rti->rti_time = 0;
+			mtx_unlock(&igmp_mtx);
 
 			timer = IGMP_MAX_HOST_REPORT_DELAY * PR_FASTHZ;
 
@@ -348,7 +374,9 @@
 		inm->inm_timer = 0;
 		inm->inm_state = IGMP_OTHERMEMBER;
 	} else {
+		mtx_lock(&igmp_mtx);
 		inm->inm_rti = find_rti(inm->inm_ifp);
+		mtx_unlock(&igmp_mtx);
 		igmp_sendpkt(inm, inm->inm_rti->rti_type, 0);
 		inm->inm_timer = IGMP_RANDOM_DELAY(
 					IGMP_MAX_HOST_REPORT_DELAY*PR_FASTHZ);
@@ -408,6 +436,7 @@
 	struct router_info *rti;
 
 	IGMP_PRINTF("[igmp.c,_slowtimo] -- > entering \n");
+	mtx_lock(&igmp_mtx);
 	SLIST_FOREACH(rti, &router_info_head, rti_list) {
 		if (rti->rti_type == IGMP_V1_ROUTER) {
 			rti->rti_time++;
@@ -415,6 +444,7 @@
 				rti->rti_type = IGMP_V2_ROUTER;
 		}
 	}
+	mtx_unlock(&igmp_mtx);
 	IGMP_PRINTF("[igmp.c,_slowtimo] -- > exiting \n");
 	splx(s);
 }
--- //depot/vendor/freebsd/src/sys/netinet/in_gif.c	2003/10/29 07:10:52
+++ //depot/user/rwatson/netperf/sys/netinet/in_gif.c	2004/03/09 15:48:47
@@ -174,6 +174,9 @@
 	}
 	bcopy(&iphdr, mtod(m, struct ip *), sizeof(struct ip));
 
+	/*
+	 * XXXRW: locking of gif's softc.
+	 */
 	if (dst->sin_family != sin_dst->sin_family ||
 	    dst->sin_addr.s_addr != sin_dst->sin_addr.s_addr) {
 		/* cache route doesn't match */
@@ -320,6 +323,10 @@
 	case 0: case 127: case 255:
 		return 0;
 	}
+
+	/*
+	 * XXXRW: Lock in_ifaddrhead walking.
+	 */
 	/* reject packets with broadcast on source */
 	TAILQ_FOREACH(ia4, &in_ifaddrhead, ia_link) {
 		if ((ia4->ia_ifa.ifa_ifp->if_flags & IFF_BROADCAST) == 0)
@@ -328,6 +335,7 @@
 			return 0;
 	}
 
+	/* XXXRW: unlocked read. */
 	/* ingress filters on outer source */
 	if ((sc->gif_if.if_flags & IFF_LINK2) == 0 && ifp) {
 		struct sockaddr_in sin;
@@ -383,6 +391,11 @@
 in_gif_attach(sc)
 	struct gif_softc *sc;
 {
+	
+	/*
+	 * XXXRW: Technically, NULL can also be returned for ENOMEM,
+	 * not just EEXIST.
+	 */
 	sc->encap_cookie4 = encap_attach_func(AF_INET, -1, gif_encapcheck,
 	    &in_gif_protosw, sc);
 	if (sc->encap_cookie4 == NULL)
--- //depot/vendor/freebsd/src/sys/netinet/in_pcb.c	2004/03/09 17:05:32
+++ //depot/user/rwatson/netperf/sys/netinet/in_pcb.c	2004/03/09 19:47:45
@@ -689,6 +689,7 @@
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
 	if (so) {
+		SOCK_LOCK(so);
 		so->so_pcb = 0;
 		sotryfree(so);
 	}
--- //depot/vendor/freebsd/src/sys/netinet/in_pcb.h	2003/11/25 17:45:19
+++ //depot/user/rwatson/netperf/sys/netinet/in_pcb.h	2004/02/29 22:51:24
@@ -248,9 +248,14 @@
 #define INP_LOCK(inp)		mtx_lock(&(inp)->inp_mtx)
 #define INP_UNLOCK(inp)		mtx_unlock(&(inp)->inp_mtx)
 #ifndef INET6
-#define INP_LOCK_ASSERT(inp)	mtx_assert(&(inp)->inp_mtx, MA_OWNED)
+#define INP_LOCK_ASSERT(inp)	do {					\
+	mtx_assert(&(inp)->inp_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #else
-#define INP_LOCK_ASSERT(inp)
+#define INP_LOCK_ASSERT(inp)	do {					\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #endif
 
 #define INP_INFO_LOCK_INIT(ipi, d) \
@@ -260,11 +265,21 @@
 #define INP_INFO_RUNLOCK(ipi)	mtx_unlock(&(ipi)->ipi_mtx)
 #define INP_INFO_WUNLOCK(ipi)	mtx_unlock(&(ipi)->ipi_mtx)
 #ifndef INET6
-#define INP_INFO_RLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED)
-#define INP_INFO_WLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED)
+#define INP_INFO_RLOCK_ASSERT(ipi)	do {				\
+	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define INP_INFO_WLOCK_ASSERT(ipi)	do {				\
+	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #else
-#define INP_INFO_RLOCK_ASSERT(ipi)
-#define INP_INFO_WLOCK_ASSERT(ipi)
+#define INP_INFO_RLOCK_ASSERT(ipi)	do {				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define INP_INFO_WLOCK_ASSERT(ipi)	do {				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #endif
 
 #define INP_PCBHASH(faddr, lport, fport, mask) \
--- //depot/vendor/freebsd/src/sys/netinet/in_proto.c	2004/02/25 11:55:45
+++ //depot/user/rwatson/netperf/sys/netinet/in_proto.c	2004/03/12 22:12:41
@@ -182,7 +182,7 @@
 { SOCK_RAW,	&inetdomain,	IPPROTO_IPV4,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
   encap4_input,	0,	 	0,		rip_ctloutput,
   0,
-  encap_init,		0,		0,		0,
+  encap_init,	0,		0,		0,
   &rip_usrreqs
 },
 { SOCK_RAW,	&inetdomain,	IPPROTO_MOBILE,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
--- //depot/vendor/freebsd/src/sys/netinet/ip_divert.c	2004/02/25 16:30:40
+++ //depot/user/rwatson/netperf/sys/netinet/ip_divert.c	2004/03/05 07:48:03
@@ -223,20 +223,6 @@
 		    sizeof(divsrc.sin_zero));
 	}
 
-	/*
-	 * XXX sbappendaddr must be protected by Giant until
-	 * we have locking at the socket layer.  When entered
-	 * from below we come in w/o Giant and must take it
-	 * here.  Unfortunately we cannot tell whether we're
-	 * entering from above (already holding Giant),
-	 * below (potentially without Giant), or otherwise
-	 * (e.g. from tcp_syncache through a timeout) so we
-	 * have to grab it regardless.  This causes a LOR with
-	 * the tcp lock, at least, and possibly others.  For
-	 * the moment we're ignoring this. Once sockets are
-	 * locked this cruft can be removed.
-	 */
-	mtx_lock(&Giant);
 	/* Put packet on socket queue, if any */
 	sa = NULL;
 	nport = htons((u_int16_t)divert_info(mtag));
@@ -258,7 +244,6 @@
 		INP_UNLOCK(inp);
 	}
 	INP_INFO_RUNLOCK(&divcbinfo);
-	mtx_unlock(&Giant);
 	if (sa == NULL) {
 		m_freem(m);
 		ipstat.ips_noproto++;
--- //depot/vendor/freebsd/src/sys/netinet/ip_dummynet.c	2004/03/02 17:35:21
+++ //depot/user/rwatson/netperf/sys/netinet/ip_dummynet.c	2004/03/03 20:00:06
@@ -171,7 +171,10 @@
 #define	DUMMYNET_LOCK_DESTROY()	mtx_destroy(&dummynet_mtx)
 #define	DUMMYNET_LOCK()		mtx_lock(&dummynet_mtx)
 #define	DUMMYNET_UNLOCK()	mtx_unlock(&dummynet_mtx)
-#define	DUMMYNET_LOCK_ASSERT()	mtx_assert(&dummynet_mtx, MA_OWNED)
+#define	DUMMYNET_LOCK_ASSERT()	do {				\
+	mtx_assert(&dummynet_mtx, MA_OWNED);			\
+	NET_ASSERT_GIANT();					\
+} while (0)
 
 static int config_pipe(struct dn_pipe *p);
 static int ip_dn_ctl(struct sockopt *sopt);
--- //depot/vendor/freebsd/src/sys/netinet/ip_encap.c	2004/03/09 18:50:38
+++ //depot/user/rwatson/netperf/sys/netinet/ip_encap.c	2004/03/09 19:47:45
@@ -1,4 +1,4 @@
-/*	$FreeBSD: src/sys/netinet/ip_encap.c,v 1.19 2004/03/10 02:48:50 rwatson Exp $	*/
+/*	$FreeBSD: src/sys/netinet/ip_encap.c,v 1.18 2003/06/01 09:20:38 phk Exp $	*/
 /*	$KAME: ip_encap.c,v 1.41 2001/03/15 08:35:08 itojun Exp $	*/
 
 /*
@@ -106,8 +106,7 @@
 LIST_HEAD(, encaptab) encaptab = LIST_HEAD_INITIALIZER(&encaptab);
 
 /*
- * We currently keey encap_init() for source code compatibility reasons --
- * it's referenced by KAME pieces in netinet6.
+ * XXXRW: encap_init() was entirely useless, so I deleted it.
  */
 void
 encap_init()
@@ -185,6 +184,10 @@
 	}
 	mtx_unlock(&encapmtx);
 
+	/*
+	 * XXXRW: Need drain mechanism to prevent the encapsulation
+	 * entry from being released while in use.
+	 */
 	if (match) {
 		/* found a match, "match" has the best one */
 		psw = match->psw;
@@ -255,6 +258,10 @@
 	}
 	mtx_unlock(&encapmtx);
 
+	/*
+	 * XXXRW: Need drain mechanism so the encap entry isn't freed
+	 * while in use.
+	 */
 	if (match) {
 		/* found a match */
 		psw = (const struct ip6protosw *)match->psw;
--- //depot/vendor/freebsd/src/sys/netinet/ip_encap.h	2002/03/19 13:30:39
+++ //depot/user/rwatson/netperf/sys/netinet/ip_encap.h	2004/03/11 22:46:13
@@ -35,6 +35,15 @@
 
 #ifdef _KERNEL
 
+/*
+ * This structure is entirely static after registration, and other than
+ * its entry in the encapsulation table, requires no locking.  The chain
+ * field is locked using the global encapmtx.
+ *
+ * XXXRW: Need to add a refcount/drain mechanism so that encapsulation
+ * entries can't be removed while in use.  This likely requires a
+ * refcount and cv to wait for it to drain, or an sx lock.
+ */
 struct encaptab {
 	LIST_ENTRY(encaptab) chain;
 	int af;
--- //depot/vendor/freebsd/src/sys/netinet/ip_fastfwd.c	2004/02/25 11:55:45
+++ //depot/user/rwatson/netperf/sys/netinet/ip_fastfwd.c	2004/02/28 14:29:37
@@ -609,6 +609,7 @@
 					    sizeof(struct sockaddr_in *),
 					    M_NOWAIT);
 					if (mtag == NULL) {
+						/* XXX statistic */
 						if (ro.ro_rt)
 							RTFREE(ro.ro_rt);
 						goto drop;
--- //depot/vendor/freebsd/src/sys/netinet/ip_fw2.c	2004/02/25 11:55:45
+++ //depot/user/rwatson/netperf/sys/netinet/ip_fw2.c	2004/02/29 22:51:24
@@ -122,7 +122,10 @@
 #define	IPFW_LOCK_DESTROY(_chain)	mtx_destroy(&(_chain)->mtx)
 #define	IPFW_LOCK(_chain)	mtx_lock(&(_chain)->mtx)
 #define	IPFW_UNLOCK(_chain)	mtx_unlock(&(_chain)->mtx)
-#define	IPFW_LOCK_ASSERT(_chain)	mtx_assert(&(_chain)->mtx, MA_OWNED)
+#define	IPFW_LOCK_ASSERT(_chain)	do {				\
+	mtx_assert(&(_chain)->mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 
 /*
  * list of rules for layer 3
@@ -1296,7 +1299,8 @@
 }
 
 static int
-check_uidgid(ipfw_insn_u32 *insn,
+check_uidgid(struct ip_fw_chain *chain,
+	ipfw_insn_u32 *insn,
 	int proto, struct ifnet *oif,
 	struct in_addr dst_ip, u_int16_t dst_port,
 	struct in_addr src_ip, u_int16_t src_port)
@@ -1317,7 +1321,10 @@
 
 	match = 0;
 
-	INP_INFO_RLOCK(pi);	/* XXX LOR with IPFW */
+	/* NB: reorder to avoid LOR between IPFW and inp */
+	IPFW_UNLOCK(chain);
+	INP_INFO_RLOCK(pi);
+	IPFW_LOCK(chain);
 	pcb =  (oif) ?
 		in_pcblookup_hash(pi,
 		    dst_ip, htons(dst_port),
@@ -1657,7 +1664,7 @@
 					break;
 				if (proto == IPPROTO_TCP ||
 				    proto == IPPROTO_UDP)
-					match = check_uidgid(
+					match = check_uidgid(chain,
 						    (ipfw_insn_u32 *)cmd,
 						    proto, oif,
 						    dst_ip, dst_port,
--- //depot/vendor/freebsd/src/sys/netinet/ip_gre.c	2004/01/30 01:05:29
+++ //depot/user/rwatson/netperf/sys/netinet/ip_gre.c	2004/03/11 22:57:45
@@ -314,6 +314,13 @@
 
 /*
  * Find the gre interface associated with our src/dst/proto set.
+ *
+ * XXXRW: Need some sort of drain/refcount mechanism so that the softc
+ * reference remains valid after it's returned from gre_lookup().  Right
+ * now, I'm thinking it should be reference-counted with a gre_dropref()
+ * when the caller is done with the softc.  This is complicated by how
+ * to handle destroying the gre softc; probably using a gre_drain() in
+ * in_gre.c during destroy.
  */
 static struct gre_softc *
 gre_lookup(m, proto)
@@ -323,14 +330,18 @@
 	struct ip *ip = mtod(m, struct ip *);
 	struct gre_softc *sc;
 
+	mtx_lock(&gre_mtx);
 	for (sc = LIST_FIRST(&gre_softc_list); sc != NULL;
 	     sc = LIST_NEXT(sc, sc_list)) {
 		if ((sc->g_dst.s_addr == ip->ip_src.s_addr) &&
 		    (sc->g_src.s_addr == ip->ip_dst.s_addr) &&
 		    (sc->g_proto == proto) &&
-		    ((sc->sc_if.if_flags & IFF_UP) != 0))
+		    ((sc->sc_if.if_flags & IFF_UP) != 0)) {
+			mtx_unlock(&gre_mtx);
 			return (sc);
+		}
 	}
+	mtx_unlock(&gre_mtx);
 
 	return (NULL);
 }
--- //depot/vendor/freebsd/src/sys/netinet/ip_id.c	2004/02/25 19:55:40
+++ //depot/user/rwatson/netperf/sys/netinet/ip_id.c	2004/03/11 19:03:57
@@ -79,6 +79,9 @@
 	2729
 };
 
+/*
+ * XXXRW: Locking?
+ */
 static u_int16_t ru_x;
 static u_int16_t ru_seed, ru_seed2;
 static u_int16_t ru_a, ru_b;
--- //depot/vendor/freebsd/src/sys/netinet/ip_input.c	2004/03/01 14:38:06
+++ //depot/user/rwatson/netperf/sys/netinet/ip_input.c	2004/03/05 07:48:03
@@ -931,14 +931,14 @@
 		/* attach next hop info for TCP */
 		struct m_tag *mtag = m_tag_get(PACKET_TAG_IPFORWARD,
 		    sizeof(struct sockaddr_in *), M_NOWAIT);
-		if (mtag == NULL)
+		if (mtag == NULL) {
+			/* XXX statistic */
 			goto bad;
+		}
 		*(struct sockaddr_in **)(mtag+1) = args.next_hop;
 		m_tag_prepend(m, mtag);
 	}
-	NET_LOCK_GIANT();
 	(*inetsw[ip_protox[ip->ip_p]].pr_input)(m, hlen);
-	NET_UNLOCK_GIANT();
 	return;
 bad:
 	m_freem(m);
@@ -1856,6 +1856,7 @@
 		struct m_tag *mtag = m_tag_get(PACKET_TAG_IPFORWARD,
 		    sizeof(struct sockaddr_in *), M_NOWAIT);
 		if (mtag == NULL) {
+			/* XXX statistic */
 			m_freem(m);
 			return;
 		}
--- //depot/vendor/freebsd/src/sys/netinet/ip_mroute.c	2004/03/07 23:51:05
+++ //depot/user/rwatson/netperf/sys/netinet/ip_mroute.c	2004/03/09 11:24:20
@@ -103,7 +103,10 @@
 static struct mtx mfc_mtx;
 #define	MFC_LOCK()	mtx_lock(&mfc_mtx)
 #define	MFC_UNLOCK()	mtx_unlock(&mfc_mtx)
-#define	MFC_LOCK_ASSERT()	mtx_assert(&mfc_mtx, MA_OWNED)
+#define	MFC_LOCK_ASSERT()	do {					\
+	mtx_assert(&mfc_mtx, MA_OWNED);					\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #define	MFC_LOCK_INIT()	mtx_init(&mfc_mtx, "mroute mfc table", NULL, MTX_DEF)
 #define	MFC_LOCK_DESTROY()	mtx_destroy(&mfc_mtx)
 
@@ -1303,13 +1306,10 @@
 socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
 {
     if (s) {
-	mtx_lock(&Giant);		/* XXX until sockets are locked */
 	if (sbappendaddr(&s->so_rcv, (struct sockaddr *)src, mm, NULL) != 0) {
 	    sorwakeup(s);
-	    mtx_unlock(&Giant);
 	    return 0;
 	}
-	mtx_unlock(&Giant);
     }
     m_freem(mm);
     return -1;
--- //depot/vendor/freebsd/src/sys/netinet/ip_output.c	2004/03/02 06:40:31
+++ //depot/user/rwatson/netperf/sys/netinet/ip_output.c	2004/03/03 20:00:06
@@ -161,6 +161,12 @@
 
 	M_ASSERTPKTHDR(m);
 	
+	/*
+	 * When packet comes from dummynet restore state from
+	 * previous processing instead of the header.  Yech!
+	 *
+	 * XXX add conditional compilation?
+	 */
 	args.next_hop = ip_claim_next_hop(m);
 	dummytag = m_tag_find(m, PACKET_TAG_DUMMYNET, NULL);
 	if (dummytag != NULL) {
@@ -181,6 +187,12 @@
 		ifp = dt->ifp;
 	}
 
+#ifdef IPSEC
+	/* XXXRW: so is not defined -- merge error from netperf+sockets? */
+	so = ipsec_getsocket(m);
+	(void)ipsec_setsocket(m, NULL);
+#endif /*IPSEC*/
+
 	if (ro == NULL) {
 		ro = &iproute;
 		bzero(ro, sizeof (*ro));
@@ -887,6 +899,7 @@
 				    PACKET_TAG_IPFORWARD,
 				    sizeof(struct sockaddr_in *), M_NOWAIT);
 				if (mtag == NULL) {
+					/* XXX statistic */
 					error = ENOBUFS;
 					goto bad;
 				}
@@ -905,6 +918,7 @@
 				    CSUM_IP_CHECKED | CSUM_IP_VALID;
 				ip->ip_len = htons(ip->ip_len);
 				ip->ip_off = htons(ip->ip_off);
+				/* XXX netisr_queue(NETISR_IP, m); */
 				ip_input(m);
 				goto done;
 			}
--- //depot/vendor/freebsd/src/sys/netinet/raw_ip.c	2004/02/17 06:05:37
+++ //depot/user/rwatson/netperf/sys/netinet/raw_ip.c	2004/03/11 19:03:33
@@ -90,6 +90,9 @@
  * so leave them not initialized and rely on BSS being set to 0.
  */
 
+/*
+ * XXXRW: Locking for mrouter bits?
+ */
 /* The socket used to communicate with the multicast routing daemon.  */
 struct socket  *ip_mrouter;
 
--- //depot/vendor/freebsd/src/sys/netinet/tcp_debug.c	2004/02/25 11:55:45
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_debug.c	2004/02/28 14:29:37
@@ -54,6 +54,7 @@
 #include <sys/systm.h>
 #include <sys/mbuf.h>
 #include <sys/protosw.h>
+#include <sys/mbuf.h>
 #include <sys/socket.h>
 
 #include <netinet/in.h>
--- //depot/vendor/freebsd/src/sys/netinet/tcp_input.c	2004/03/01 11:15:19
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_input.c	2004/03/01 16:04:44
@@ -428,7 +428,7 @@
 	struct tcpopt to;		/* options in this segment */
 	struct rmxp_tao tao;		/* our TAO cache entry */
 	int headlocked = 0;
-	struct sockaddr_in *next_hop = NULL;
+	struct sockaddr_in *next_hop;
 	int rstreason; /* For badport_bandlim accounting purposes */
 
 	struct ip6_hdr *ip6 = NULL;
@@ -1166,6 +1166,7 @@
 				acked = th->th_ack - tp->snd_una;
 				tcpstat.tcps_rcvackpack++;
 				tcpstat.tcps_rcvackbyte += acked;
+				SOCKBUF_LOCK(&so->so_snd);
 				sbdrop(&so->so_snd, acked);
 				if (SEQ_GT(tp->snd_una, tp->snd_recover) &&
 				    SEQ_LEQ(th->th_ack, tp->snd_recover))
@@ -1203,7 +1204,8 @@
 						      tp->t_rxtcur,
 						      tcp_timer_rexmt, tp);
 
-				sowwakeup(so);
+				sowwakeup_locked(so);
+				SOCKBUF_UNLOCK(&so->so_snd);
 				if (so->so_snd.sb_cc)
 					(void) tcp_output(tp);
 				goto check_delack;
@@ -2093,6 +2095,7 @@
 				incr = incr * incr / cw;
 			tp->snd_cwnd = min(cw+incr, TCP_MAXWIN<<tp->snd_scale);
 		}
+		SOCKBUF_LOCK(&so->so_snd);
 		if (acked > so->so_snd.sb_cc) {
 			tp->snd_wnd -= so->so_snd.sb_cc;
 			sbdrop(&so->so_snd, (int)so->so_snd.sb_cc);
@@ -2102,7 +2105,8 @@
 			tp->snd_wnd -= acked;
 			ourfinisacked = 0;
 		}
-		sowwakeup(so);
+		sowwakeup_locked(so);
+		SOCKBUF_UNLOCK(&so->so_snd);
 		/* detect una wraparound */
 		if (tcp_do_newreno && !IN_FASTRECOVERY(tp) &&
 		    SEQ_GT(tp->snd_una, tp->snd_recover) &&
--- //depot/vendor/freebsd/src/sys/netinet/tcp_subr.c	2004/02/28 07:15:35
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_subr.c	2004/03/05 23:20:26
@@ -560,6 +560,7 @@
 #ifdef INET6
 	int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
 #endif /* INET6 */
+	int callout_flag;
 
 	tm = uma_zalloc(tcpcb_zone, M_NOWAIT | M_ZERO);
 	if (tm == NULL)
@@ -573,11 +574,17 @@
 		tcp_mssdflt;
 
 	/* Set up our timeouts. */
-	callout_init(tp->tt_rexmt = &tm->tcpcb_mem_rexmt, 0);
-	callout_init(tp->tt_persist = &tm->tcpcb_mem_persist, 0);
-	callout_init(tp->tt_keep = &tm->tcpcb_mem_keep, 0);
-	callout_init(tp->tt_2msl = &tm->tcpcb_mem_2msl, 0);
-	callout_init(tp->tt_delack = &tm->tcpcb_mem_delack, 0);
+	/*
+	 * XXXRW: Are these actually MPSAFE?  I think so, but need to
+	 * review the timed wait code, as it has some list variables,
+	 * etc, that are global.
+	 */
+	callout_flag = debug_mpsafenet ? CALLOUT_MPSAFE : 0;
+	callout_init(tp->tt_rexmt = &tm->tcpcb_mem_rexmt, callout_flag);
+	callout_init(tp->tt_persist = &tm->tcpcb_mem_persist, callout_flag);
+	callout_init(tp->tt_keep = &tm->tcpcb_mem_keep, callout_flag);
+	callout_init(tp->tt_2msl = &tm->tcpcb_mem_2msl, callout_flag);
+	callout_init(tp->tt_delack = &tm->tcpcb_mem_delack, callout_flag);
 
 	if (tcp_do_rfc1323)
 		tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
@@ -1542,7 +1549,7 @@
 
 /*
  * Move a TCP connection into TIME_WAIT state.
- *    tcbinfo is unlocked.
+ *    tcbinfo is locked.
  *    inp is locked, and is unlocked before returning.
  */
 void
@@ -1554,6 +1561,11 @@
 	int tw_time, acknow;
 	struct socket *so;
 
+	INP_INFO_WLOCK_ASSERT(&tcbinfo);
+#if 0
+	INP_LOCK_ASSERT(tp);
+#endif
+
 	tw = uma_zalloc(tcptw_zone, M_NOWAIT);
 	if (tw == NULL) {
 		tw = tcp_timer_2msl_tw(1);
@@ -1604,13 +1616,19 @@
 	}
 	tcp_discardcb(tp);
 	so = inp->inp_socket;
+	SOCK_LOCK(so);
 	so->so_pcb = NULL;
 	tw->tw_cred = crhold(so->so_cred);
 	tw->tw_so_options = so->so_options;
+	sotryfree(so);			/* NB: drops lock */
+	inp->inp_socket = NULL;
 	if (acknow)
 		tcp_twrespond(tw, TH_ACK);
+#if 0
+	/* XXXRW: Sam removed this, need to check why. */
 	sotryfree(so);
 	inp->inp_socket = NULL;
+#endif
 	inp->inp_ppcb = (caddr_t)tw;
 	inp->inp_vflag |= INP_TIMEWAIT;
 	tcp_timer_2msl_reset(tw, tw_time);
@@ -1686,6 +1704,8 @@
 	int isipv6 = inp->inp_inc.inc_isipv6;
 #endif
 
+	INP_LOCK_ASSERT(inp);
+
 	m = m_gethdr(M_DONTWAIT, MT_HEADER);
 	if (m == NULL)
 		return (ENOBUFS);
--- //depot/vendor/freebsd/src/sys/netinet/tcp_syncache.c	2004/02/14 13:50:39
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_syncache.c	2004/03/01 18:05:01
@@ -540,7 +540,7 @@
 	struct socket *so;
 	struct tcpcb *tp;
 
-	GIANT_REQUIRED;			/* XXX until socket locking */
+	NET_ASSERT_GIANT();
 	INP_INFO_WLOCK_ASSERT(&tcbinfo);
 
 	/*
--- //depot/vendor/freebsd/src/sys/netinet/tcp_timer.c	2003/11/20 12:10:44
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_timer.c	2004/03/06 21:55:09
@@ -273,6 +273,9 @@
 	}
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 void
 tcp_timer_2msl_reset(struct tcptw *tw, int timeo)
 {
@@ -287,6 +290,9 @@
 	LIST_INSERT_BEFORE(tw_tail, tw, tw_2msl);
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 void
 tcp_timer_2msl_stop(struct tcptw *tw)
 {
@@ -295,6 +301,9 @@
 		LIST_REMOVE(tw, tw_2msl);
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 struct tcptw *
 tcp_timer_2msl_tw(int reuse)
 {
--- //depot/vendor/freebsd/src/sys/netinet/tcp_usrreq.c	2004/02/16 14:25:21
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_usrreq.c	2004/02/28 14:29:37
@@ -120,7 +120,6 @@
 static int
 tcp_usr_attach(struct socket *so, int proto, struct thread *td)
 {
-	int s = splnet();
 	int error;
 	struct inpcb *inp;
 	struct tcpcb *tp = 0;
@@ -146,11 +145,71 @@
 out:
 	TCPDEBUG2(PRU_ATTACH);
 	INP_INFO_WUNLOCK(&tcbinfo);
-	splx(s);
 	return error;
 }
 
 /*
+ * Common code to setup and teardown locking.  Most
+ * code begins with a COMMON_START macro and finishes
+ * with COMMON_END.  You indicate whether the inpcb
+ * and enclosing head are to be locked read or write 
+ * and whether there is an existing sockbuf lock that
+ * needs to be re-ordered.
+ */
+#define INI_NOLOCK	0		/* no head lock */
+#define INI_READ	1		/* read head lock */
+#define INI_WRITE	2		/* write head lock */
+#define SBI_NONE	0		/* no sockbuf lock to reorder */
+#define SBI_SND		1		/* reorder so->so_snd lock */
+#define SBI_RCV		2		/* reorder so->so_rcv lock */
+
+#define	COMMON_START0(_headrw, _sbrw) do {			\
+	if (_sbrw == SBI_SND)					\
+		SOCKBUF_UNLOCK(&so->so_snd);			\
+	else if (_sbrw == SBI_RCV)				\
+		SOCKBUF_UNLOCK(&so->so_rcv);			\
+	if (_headrw == INI_READ)				\
+		INP_INFO_RLOCK(&tcbinfo);			\
+	else if (_headrw == INI_WRITE)				\
+		INP_INFO_WLOCK(&tcbinfo);			\
+	inp = sotoinpcb(so);					\
+	if (inp == 0) {						\
+		if (_sbrw == SBI_SND)				\
+			SOCKBUF_LOCK(&so->so_snd);		\
+		else if (_sbrw == SBI_RCV)			\
+			SOCKBUF_LOCK(&so->so_rcv);		\
+		if (_headrw == INI_READ)			\
+			INP_INFO_RUNLOCK(&tcbinfo);		\
+		else if (_headrw == INI_WRITE)			\
+			INP_INFO_WUNLOCK(&tcbinfo);		\
+		return EINVAL;					\
+	}							\
+	INP_LOCK(inp);						\
+	if (_sbrw == SBI_SND)					\
+		SOCKBUF_LOCK(&so->so_snd);			\
+	else if (_sbrw == SBI_RCV)				\
+		SOCKBUF_LOCK(&so->so_rcv);			\
+	if (_headrw == INI_READ)				\
+		INP_INFO_RUNLOCK(&tcbinfo);			\
+	tp = intotcpcb(inp);					\
+	TCPDEBUG1();						\
+} while(0)
+
+#define	COMMON_START(_headrw, _sbrw) do {			\
+	TCPDEBUG0;						\
+	COMMON_START0(_headrw, _sbrw);				\
+} while (0)
+
+#define COMMON_END(_headrw, req)				\
+	TCPDEBUG2(req);						\
+	do {							\
+		if (tp)						\
+			INP_UNLOCK(inp);			\
+		if (_headrw == INI_WRITE)			\
+			INP_INFO_WUNLOCK(&tcbinfo);		\
+	} while(0)
+
+/*
  * pru_detach() detaches the TCP protocol from the socket.
  * If the protocol state is non-embryonic, then can't
  * do this directly: have to initiate a pru_disconnect(),
@@ -160,85 +219,28 @@
 static int
 tcp_usr_detach(struct socket *so)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	TCPDEBUG0;
 
-	INP_INFO_WLOCK(&tcbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&tcbinfo);
-		splx(s);
-		return EINVAL;	/* XXX */
-	}
-	INP_LOCK(inp);
-	tp = intotcpcb(inp);
-	TCPDEBUG1();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	tp = tcp_disconnect(tp);
-
-	TCPDEBUG2(PRU_DETACH);
-	if (tp)
-		INP_UNLOCK(inp);
-	INP_INFO_WUNLOCK(&tcbinfo);
-	splx(s);
+	COMMON_END(INI_WRITE, PRU_DETACH);
 	return error;
 }
 
-#define INI_NOLOCK	0
-#define INI_READ	1
-#define INI_WRITE	2
-
-#define	COMMON_START()						\
-	TCPDEBUG0;						\
-	do {							\
-		if (inirw == INI_READ)				\
-			INP_INFO_RLOCK(&tcbinfo);		\
-		else if (inirw == INI_WRITE)			\
-			INP_INFO_WLOCK(&tcbinfo);		\
-		inp = sotoinpcb(so);				\
-		if (inp == 0) {					\
-			if (inirw == INI_READ)			\
-				INP_INFO_RUNLOCK(&tcbinfo);	\
-			else if (inirw == INI_WRITE)		\
-				INP_INFO_WUNLOCK(&tcbinfo);	\
-			splx(s);				\
-			return EINVAL;				\
-		}						\
-		INP_LOCK(inp);					\
-		if (inirw == INI_READ)				\
-			INP_INFO_RUNLOCK(&tcbinfo);		\
-		tp = intotcpcb(inp);				\
-		TCPDEBUG1();					\
-} while(0)
-
-#define COMMON_END(req)						\
-out:	TCPDEBUG2(req);						\
-	do {							\
-		if (tp)						\
-			INP_UNLOCK(inp);			\
-		if (inirw == INI_WRITE)				\
-			INP_INFO_WUNLOCK(&tcbinfo);		\
-		splx(s);					\
-		return error;					\
-		goto out;					\
-} while(0)
-
 /*
  * Give the socket an address.
  */
 static int
 tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct sockaddr_in *sinp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 
 	/*
 	 * Must check for multicast addresses and disallow binding
@@ -253,23 +255,21 @@
 		goto out;
 	}
 	error = in_pcbbind(inp, nam, td);
-	if (error)
-		goto out;
-	COMMON_END(PRU_BIND);
+out:
+	COMMON_END(INI_WRITE, PRU_BIND);
+	return error;
 }
 
 #ifdef INET6
 static int
 tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct sockaddr_in6 *sin6p;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 
 	/*
 	 * Must check for multicast addresses and disallow binding
@@ -299,9 +299,9 @@
 		}
 	}
 	error = in6_pcbbind(inp, nam, td);
-	if (error)
-		goto out;
-	COMMON_END(PRU_BIND);
+out:
+	COMMON_END(INI_WRITE, PRU_BIND);
+	return error;
 }
 #endif /* INET6 */
 
@@ -311,31 +311,28 @@
 static int
 tcp_usr_listen(struct socket *so, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	if (inp->inp_lport == 0)
 		error = in_pcbbind(inp, (struct sockaddr *)0, td);
 	if (error == 0)
 		tp->t_state = TCPS_LISTEN;
-	COMMON_END(PRU_LISTEN);
+	COMMON_END(INI_WRITE, PRU_LISTEN);
+	return error;
 }
 
 #ifdef INET6
 static int
 tcp6_usr_listen(struct socket *so, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	if (inp->inp_lport == 0) {
 		inp->inp_vflag &= ~INP_IPV4;
 		if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0)
@@ -344,7 +341,8 @@
 	}
 	if (error == 0)
 		tp->t_state = TCPS_LISTEN;
-	COMMON_END(PRU_LISTEN);
+	COMMON_END(INI_WRITE, PRU_LISTEN);
+	return error;
 }
 #endif /* INET6 */
 
@@ -358,14 +356,12 @@
 static int
 tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct sockaddr_in *sinp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 
 	/*
 	 * Must disallow TCP ``connections'' to multicast addresses.
@@ -385,21 +381,21 @@
 	if ((error = tcp_connect(tp, nam, td)) != 0)
 		goto out;
 	error = tcp_output(tp);
-	COMMON_END(PRU_CONNECT);
+out:
+	COMMON_END(INI_WRITE, PRU_CONNECT);
+	return error;
 }
 
 #ifdef INET6
 static int
 tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct sockaddr_in6 *sin6p;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 
 	/*
 	 * Must disallow TCP ``connections'' to multicast addresses.
@@ -435,7 +431,9 @@
 	if ((error = tcp6_connect(tp, nam, td)) != 0)
 		goto out;
 	error = tcp_output(tp);
-	COMMON_END(PRU_CONNECT);
+out:
+	COMMON_END(INI_WRITE, PRU_CONNECT);
+	return error;
 }
 #endif /* INET6 */
 
@@ -453,15 +451,14 @@
 static int
 tcp_usr_disconnect(struct socket *so)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	tp = tcp_disconnect(tp);
-	COMMON_END(PRU_DISCONNECT);
+	COMMON_END(INI_WRITE, PRU_DISCONNECT);
+	return error;
 }
 
 /*
@@ -472,7 +469,6 @@
 static int
 tcp_usr_accept(struct socket *so, struct sockaddr **nam)
 {
-	int s;
 	int error = 0;
 	struct inpcb *inp = NULL;
 	struct tcpcb *tp = NULL;
@@ -482,34 +478,21 @@
 
 	if (so->so_state & SS_ISDISCONNECTED) {
 		error = ECONNABORTED;
-		goto out;
+		goto out;	/* NB: ok 'cuz tp is NULL */
 	}
 
-	s = splnet();
-	INP_INFO_RLOCK(&tcbinfo);
-	inp = sotoinpcb(so);
-	if (!inp) {
-		INP_INFO_RUNLOCK(&tcbinfo);
-		splx(s);
-		return (EINVAL);
-	}
-	INP_LOCK(inp);
-	INP_INFO_RUNLOCK(&tcbinfo);
-	tp = intotcpcb(inp);
-	TCPDEBUG1();
+	COMMON_START0(INI_READ, SBI_NONE);
 
 	/* 
-	 * We inline in_setpeeraddr and COMMON_END here, so that we can
-	 * copy the data of interest and defer the malloc until after we
-	 * release the lock.
+	 * We inline in_setpeeraddr so that we can copy the
+	 * data of interest and defer the malloc until after
+	 * we release the lock.
 	 */
 	port = inp->inp_fport;
 	addr = inp->inp_faddr;
 
-out:	TCPDEBUG2(PRU_ACCEPT);
-	if (tp)
-		INP_UNLOCK(inp);
-	splx(s);
+out:
+	COMMON_END(INI_READ, PRU_ACCEPT);
 	if (error == 0)
 		*nam = in_sockaddr(port, &addr);
 	return error;
@@ -519,7 +502,6 @@
 static int
 tcp6_usr_accept(struct socket *so, struct sockaddr **nam)
 {
-	int s;
 	struct inpcb *inp = NULL;
 	int error = 0;
 	struct tcpcb *tp = NULL;
@@ -531,25 +513,14 @@
 
 	if (so->so_state & SS_ISDISCONNECTED) {
 		error = ECONNABORTED;
-		goto out;
+		goto out;		/* NB: ok 'cuz tp is NULL */
 	}
 
-	s = splnet();
-	INP_INFO_RLOCK(&tcbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_RUNLOCK(&tcbinfo);
-		splx(s);
-		return (EINVAL);
-	}
-	INP_LOCK(inp);
-	INP_INFO_RUNLOCK(&tcbinfo);
-	tp = intotcpcb(inp);
-	TCPDEBUG1();
+	COMMON_START0(INI_READ, SBI_NONE);
 	/* 
-	 * We inline in6_mapped_peeraddr and COMMON_END here, so that we can
-	 * copy the data of interest and defer the malloc until after we
-	 * release the lock.
+	 * We inline in6_mapped_peeraddr so that we can
+	 * copy the data of interest and defer the malloc
+	 * until after we release the lock.
 	 */
 	if (inp->inp_vflag & INP_IPV4) {
 		v4 = 1;
@@ -560,10 +531,8 @@
 		addr6 = inp->in6p_faddr;
 	}
 
-out:	TCPDEBUG2(PRU_ACCEPT);
-	if (tp)
-		INP_UNLOCK(inp);
-	splx(s);
+out:
+	COMMON_END(INI_READ, PRU_ACCEPT);
 	if (error == 0) {
 		if (v4)
 			*nam = in6_v4mapsin6_sockaddr(port, &addr);
@@ -601,18 +570,17 @@
 static int
 tcp_usr_shutdown(struct socket *so)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	socantsendmore(so);
 	tp = tcp_usrclosed(tp);
 	if (tp)
 		error = tcp_output(tp);
-	COMMON_END(PRU_SHUTDOWN);
+	COMMON_END(INI_WRITE, PRU_SHUTDOWN);
+	return error;
 }
 
 /*
@@ -621,15 +589,14 @@
 static int
 tcp_usr_rcvd(struct socket *so, int flags)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_READ;
 
-	COMMON_START();
+	COMMON_START(INI_READ, SBI_RCV);
 	tcp_output(tp);
-	COMMON_END(PRU_RCVD);
+	COMMON_END(INI_READ, PRU_RCVD);
+	return error;
 }
 
 /*
@@ -643,11 +610,9 @@
 tcp_usr_send(struct socket *so, int flags, struct mbuf *m, 
 	     struct sockaddr *nam, struct mbuf *control, struct thread *td)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 #ifdef INET6
 	int isipv6;
 #endif
@@ -659,9 +624,11 @@
 	 * We really want to have to this function upgrade from read lock
 	 * to write lock.  XXX
 	 */
+	SOCKBUF_UNLOCK(&so->so_snd);
 	INP_INFO_WLOCK(&tcbinfo);
 	inp = sotoinpcb(so);
 	if (inp == NULL) {
+		SOCKBUF_LOCK(&so->so_snd);
 		/*
 		 * OOPS! we lost a race, the TCP session got reset after
 		 * we checked SS_CANTSENDMORE, eg: while doing uiomove or a
@@ -677,6 +644,7 @@
 		goto out;
 	}
 	INP_LOCK(inp);
+	SOCKBUF_LOCK(&so->so_snd);
 #ifdef INET6
 	isipv6 = nam && nam->sa_family == AF_INET6;
 #endif /* INET6 */
@@ -767,8 +735,10 @@
 		error = tcp_output(tp);
 		tp->t_force = 0;
 	}
-	COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : 
+out:
+	COMMON_END(INI_WRITE, (flags & PRUS_OOB) ? PRU_SENDOOB : 
 		   ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
+	return error;
 }
 
 /*
@@ -777,15 +747,14 @@
 static int
 tcp_usr_abort(struct socket *so)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_WRITE;
 
-	COMMON_START();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	tp = tcp_drop(tp, ECONNABORTED);
-	COMMON_END(PRU_ABORT);
+	COMMON_END(INI_WRITE, PRU_ABORT);
+	return error;
 }
 
 /*
@@ -794,13 +763,11 @@
 static int
 tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
 {
-	int s = splnet();
 	int error = 0;
 	struct inpcb *inp;
 	struct tcpcb *tp;
-	const int inirw = INI_READ;
 
-	COMMON_START();
+	COMMON_START(INI_READ, SBI_NONE);
 	if ((so->so_oobmark == 0 &&
 	     (so->so_state & SS_RCVATMARK) == 0) ||
 	    so->so_options & SO_OOBINLINE ||
@@ -816,7 +783,9 @@
 	*mtod(m, caddr_t) = tp->t_iobc;
 	if ((flags & MSG_PEEK) == 0)
 		tp->t_oobflags ^= (TCPOOB_HAVEDATA | TCPOOB_HADDATA);
-	COMMON_END(PRU_RCVOOB);
+out:
+	COMMON_END(INI_READ, PRU_RCVOOB);
+	return error;
 }
 
 /* xxx - should be const */
@@ -1035,17 +1004,15 @@
 	struct socket *so;
 	struct sockopt *sopt;
 {
-	int	error, opt, optval, s;
+	int	error, opt, optval;
 	struct	inpcb *inp;
 	struct	tcpcb *tp;
 
 	error = 0;
-	s = splnet();		/* XXX */
 	INP_INFO_RLOCK(&tcbinfo);
 	inp = sotoinpcb(so);
 	if (inp == NULL) {
 		INP_INFO_RUNLOCK(&tcbinfo);
-		splx(s);
 		return (ECONNRESET);
 	}
 	INP_LOCK(inp);
@@ -1058,7 +1025,6 @@
 #endif /* INET6 */
 		error = ip_ctloutput(so, sopt);
 		INP_UNLOCK(inp);
-		splx(s);
 		return (error);
 	}
 	tp = intotcpcb(inp);
@@ -1165,7 +1131,6 @@
 		break;
 	}
 	INP_UNLOCK(inp);
-	splx(s);
 	return (error);
 }
 
@@ -1253,7 +1218,9 @@
 		tp = tcp_drop(tp, 0);
 	else {
 		soisdisconnecting(so);
+		SOCKBUF_LOCK(&so->so_rcv);
 		sbflush(&so->so_rcv);
+		SOCKBUF_UNLOCK(&so->so_rcv);
 		tp = tcp_usrclosed(tp);
 		if (tp)
 			(void) tcp_output(tp);
@@ -1306,4 +1273,3 @@
 	}
 	return (tp);
 }
-
--- //depot/vendor/freebsd/src/sys/netinet/udp_usrreq.c	2004/02/25 16:30:40
+++ //depot/user/rwatson/netperf/sys/netinet/udp_usrreq.c	2004/02/28 14:29:37
@@ -901,24 +901,67 @@
 SYSCTL_INT(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW,
     &udp_recvspace, 0, "Maximum space for incoming UDP datagrams");
 
+/*
+ * Common code to setup and teardown locking.  Most
+ * code begins with a COMMON_START macro and finishes
+ * with COMMON_END.  You indicate whether the inpcb
+ * and enclosing head are to be locked read or write 
+ * and whether there is an existing sockbuf lock that
+ * needs to be re-ordered.
+ */
+#define INI_NOLOCK	0		/* no head lock */
+#define INI_READ	1		/* read head lock */
+#define INI_WRITE	2		/* write head lock */
+#define SBI_NONE	0		/* no sockbuf lock to reorder */
+#define SBI_SND		1		/* reorder so->so_snd lock */
+#define SBI_RCV		2		/* reorder so->so_rcv lock */
+
+#define	COMMON_START(_headrw, _sbrw) do {			\
+	if (_sbrw == SBI_SND)					\
+		SOCKBUF_UNLOCK(&so->so_snd);			\
+	else if (_sbrw == SBI_RCV)				\
+		SOCKBUF_UNLOCK(&so->so_rcv);			\
+	if (_headrw == INI_READ)				\
+		INP_INFO_RLOCK(&udbinfo);			\
+	else if (_headrw == INI_WRITE)				\
+		INP_INFO_WLOCK(&udbinfo);			\
+	inp = sotoinpcb(so);					\
+	if (inp == 0) {						\
+		if (_sbrw == SBI_SND)				\
+			SOCKBUF_LOCK(&so->so_snd);		\
+		else if (_sbrw == SBI_RCV)			\
+			SOCKBUF_LOCK(&so->so_rcv);		\
+		if (_headrw == INI_READ)			\
+			INP_INFO_RUNLOCK(&udbinfo);		\
+		else if (_headrw == INI_WRITE)			\
+			INP_INFO_WUNLOCK(&udbinfo);		\
+		return EINVAL;					\
+	}							\
+	INP_LOCK(inp);						\
+	if (_sbrw == SBI_SND)					\
+		SOCKBUF_LOCK(&so->so_snd);			\
+	else if (_sbrw == SBI_RCV)				\
+		SOCKBUF_LOCK(&so->so_rcv);			\
+	if (_headrw == INI_READ)				\
+		INP_INFO_RUNLOCK(&udbinfo);			\
+} while(0)
+
+#define COMMON_END(_headrw)					\
+	do {							\
+		INP_UNLOCK(inp);				\
+		if (_headrw == INI_WRITE)			\
+			INP_INFO_WUNLOCK(&udbinfo);		\
+	} while(0)
+
 static int
 udp_abort(struct socket *so)
 {
 	struct inpcb *inp;
-	int s;
 
-	INP_INFO_WLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EINVAL;	/* ??? possible? panic instead? */
-	}
-	INP_LOCK(inp);
+	COMMON_START(INI_WRITE, SBI_NONE);
 	soisdisconnected(so);
-	s = splnet();
 	in_pcbdetach(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
-	splx(s);
 	return 0;
 }
 
@@ -960,20 +1003,11 @@
 udp_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
 	struct inpcb *inp;
-	int s, error;
+	int error;
 
-	INP_INFO_WLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EINVAL;
-	}
-	INP_LOCK(inp);
-	s = splnet();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	error = in_pcbbind(inp, nam, td);
-	splx(s);
-	INP_UNLOCK(inp);
-	INP_INFO_WUNLOCK(&udbinfo);
+	COMMON_END(INI_WRITE);
 	return error;
 }
 
@@ -981,31 +1015,22 @@
 udp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 {
 	struct inpcb *inp;
-	int s, error;
+	int error;
 	struct sockaddr_in *sin;
 
-	INP_INFO_WLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EINVAL;
-	}
-	INP_LOCK(inp);
+	COMMON_START(INI_WRITE, SBI_NONE);
 	if (inp->inp_faddr.s_addr != INADDR_ANY) {
-		INP_UNLOCK(inp);
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EISCONN;
+		error = EISCONN;
+		goto out;
 	}
-	s = splnet();
 	sin = (struct sockaddr_in *)nam;
 	if (td && jailed(td->td_ucred))
 		prison_remote_ip(td->td_ucred, 0, &sin->sin_addr.s_addr);
 	error = in_pcbconnect(inp, nam, td);
-	splx(s);
 	if (error == 0)
 		soisconnected(so);
-	INP_UNLOCK(inp);
-	INP_INFO_WUNLOCK(&udbinfo);
+out:
+	COMMON_END(INI_WRITE);
 	return error;
 }
 
@@ -1013,49 +1038,31 @@
 udp_detach(struct socket *so)
 {
 	struct inpcb *inp;
-	int s;
 
-	INP_INFO_WLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EINVAL;
-	}
-	INP_LOCK(inp);
-	s = splnet();
+	COMMON_START(INI_WRITE, SBI_NONE);
 	in_pcbdetach(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
-	splx(s);
 	return 0;
 }
 
 static int
 udp_disconnect(struct socket *so)
 {
+	int error = 0;
 	struct inpcb *inp;
-	int s;
 
-	INP_INFO_WLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		return EINVAL;
-	}
-	INP_LOCK(inp);
+	COMMON_START(INI_WRITE, SBI_NONE);
 	if (inp->inp_faddr.s_addr == INADDR_ANY) {
-		INP_INFO_WUNLOCK(&udbinfo);
-		INP_UNLOCK(inp);
-		return ENOTCONN;
+		error = ENOTCONN;
+		goto out;
 	}
 
-	s = splnet();
 	in_pcbdisconnect(inp);
 	inp->inp_laddr.s_addr = INADDR_ANY;
-	INP_UNLOCK(inp);
-	INP_INFO_WUNLOCK(&udbinfo);
-	splx(s);
+	COMMON_END(INI_WRITE);
 	so->so_state &= ~SS_ISCONNECTED;		/* XXX */
-	return 0;
+out:
+	return error;
 }
 
 static int
@@ -1063,20 +1070,23 @@
 	    struct mbuf *control, struct thread *td)
 {
 	struct inpcb *inp;
-	int ret;
+	int error;
 
+	SOCKBUF_UNLOCK(&so->so_snd);
 	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
 	if (inp == 0) {
+		SOCKBUF_LOCK(&so->so_snd);
 		INP_INFO_WUNLOCK(&udbinfo);
 		m_freem(m);
 		return EINVAL;
 	}
 	INP_LOCK(inp);
-	ret = udp_output(inp, m, addr, control, td);
+	SOCKBUF_LOCK(&so->so_snd);
+	error = udp_output(inp, m, addr, control, td);
 	INP_UNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
-	return ret; 
+	return error; 
 }
 
 int
@@ -1084,16 +1094,9 @@
 {
 	struct inpcb *inp;
 
-	INP_INFO_RLOCK(&udbinfo);
-	inp = sotoinpcb(so);
-	if (inp == 0) {
-		INP_INFO_RUNLOCK(&udbinfo);
-		return EINVAL;
-	}
-	INP_LOCK(inp);
-	INP_INFO_RUNLOCK(&udbinfo);
+	COMMON_START(INI_READ, SBI_NONE);
 	socantsendmore(so);
-	INP_UNLOCK(inp);
+	COMMON_END(INI_READ);
 	return 0;
 }
 
--- //depot/vendor/freebsd/src/sys/netinet6/in6_gif.c	2003/10/29 07:10:52
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_gif.c	2004/03/09 15:48:47
@@ -82,6 +82,11 @@
   &rip6_usrreqs
 };
 
+/*
+ * XXXRW: in6_gif per-softc locking required.  Need to lock both the
+ * members, and also prevent the softc from disappearing during use
+ * including the route).
+ */
 int
 in6_gif_output(ifp, family, m)
 	struct ifnet *ifp;
@@ -379,6 +384,10 @@
 in6_gif_attach(sc)
 	struct gif_softc *sc;
 {
+
+	/*
+	 * XXXRW: Technically, encap_attach() can return NULL due to ENOMEM?
+	 */
 	sc->encap_cookie6 = encap_attach_func(AF_INET6, -1, gif_encapcheck,
 	    (struct protosw *)&in6_gif_protosw, sc);
 	if (sc->encap_cookie6 == NULL)
--- //depot/vendor/freebsd/src/sys/netinet6/in6_ifattach.c	2004/02/25 19:55:40
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_ifattach.c	2004/03/07 10:23:16
@@ -226,8 +226,8 @@
 	struct sockaddr_dl *sdl;
 	u_int8_t *addr;
 	size_t addrlen;
-	static u_int8_t allzero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
-	static u_int8_t allone[8] =
+	static const u_int8_t allzero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+	static const u_int8_t allone[8] =
 		{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 	for (ifa = ifp->if_addrlist.tqh_first;
--- //depot/vendor/freebsd/src/sys/netinet6/in6_pcb.c	2004/02/13 06:50:50
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_pcb.c	2004/02/28 14:29:37
@@ -442,6 +442,7 @@
 	in_pcbremlists(inp);
 
 	if (so) {
+		SOCK_LOCK(so);
 		so->so_pcb = NULL;
 		sotryfree(so);
 	}
--- //depot/vendor/freebsd/src/sys/netinet6/in6_prefix.c	2003/10/06 07:06:01
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_prefix.c	2004/03/07 11:31:09
@@ -100,6 +100,8 @@
 static int link_stray_ia6s __P((struct rr_prefix *rpp));
 static void	rp_remove __P((struct rr_prefix *rpp));
 
+static int	delete_each_prefix __P((struct rr_prefix *rpp, u_char origin));
+
 /*
  * Copy bits from src to tgt, from off bit for len bits.
  * Caller must specify collect tgtsize and srcsize.
@@ -955,7 +957,7 @@
 	}
 }
 
-int
+static int
 delete_each_prefix(struct rr_prefix *rpp, u_char origin)
 {
 	int error = 0;
--- //depot/vendor/freebsd/src/sys/netinet6/in6_prefix.h	2001/07/17 00:20:42
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_prefix.h	2004/03/07 11:31:09
@@ -88,4 +88,3 @@
 
 void in6_rr_timer __P((void *));
 extern struct callout in6_rr_timer_ch;
-int delete_each_prefix  __P((struct rr_prefix *rpp, u_char origin));
--- //depot/vendor/freebsd/src/sys/netinet6/in6_proto.c	2003/11/04 12:23:18
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_proto.c	2004/03/12 22:18:52
@@ -71,6 +71,8 @@
 #include "opt_random_ip_id.h"
 
 #include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/protosw.h>
--- //depot/vendor/freebsd/src/sys/netinet6/in6_rmx.c	2003/11/20 12:10:44
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_rmx.c	2004/02/28 14:29:37
@@ -79,6 +79,8 @@
 #include <sys/sysctl.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/socketvar.h>
 #include <sys/mbuf.h>
 #include <sys/syslog.h>
--- //depot/vendor/freebsd/src/sys/netinet6/nd6.c	2004/01/28 07:06:38
+++ //depot/user/rwatson/netperf/sys/netinet6/nd6.c	2004/03/11 19:06:45
@@ -98,6 +98,9 @@
 /* for debugging? */
 static int nd6_inuse, nd6_allocated;
 
+/*
+ * XXXRW: What follows requires locking.
+ */
 struct llinfo_nd6 llinfo_nd6 = {&llinfo_nd6, &llinfo_nd6};
 struct nd_drhead nd_defrouter;
 struct nd_prhead nd_prefix = { 0 };
--- //depot/vendor/freebsd/src/sys/netipx/ipx_input.c	2003/11/08 14:30:39
+++ //depot/user/rwatson/netperf/sys/netipx/ipx_input.c	2004/03/11 19:09:53
@@ -72,23 +72,32 @@
 SYSCTL_INT(_net_ipx, OID_AUTO, ipxnetbios, CTLFLAG_RW,
 	   &ipxnetbios, 0, "");
 
-union	ipx_net ipx_zeronet;
-union	ipx_host ipx_zerohost;
+const union	ipx_net ipx_zeronet;
+const union	ipx_host ipx_zerohost;
 
 union	ipx_net	ipx_broadnet;
 union	ipx_host ipx_broadhost;
 
+/*
+ * XXXRW: Locking needed here.
+ */
 struct	ipxstat ipxstat;
 struct	sockaddr_ipx ipx_netmask, ipx_hostmask;
 
-static	u_short allones[] = {-1, -1, -1};
+static	const u_short allones[] = {-1, -1, -1};
 
+/*
+ * XXXRW: Locking needed here.
+ */
 struct	ipxpcb ipxpcb;
 struct	ipxpcb ipxrawpcb;
 
 static int ipxqmaxlen = IFQ_MAXLEN;
 static	struct ifqueue ipxintrq;
 
+/*
+ * XXXRW: Locking needed here.
+ */
 long	ipx_pexseq;
 
 static	int ipx_do_route(struct ipx_addr *src, struct route *ro);
@@ -103,8 +112,8 @@
 void
 ipx_init()
 {
-	ipx_broadnet = *(union ipx_net *)allones;
-	ipx_broadhost = *(union ipx_host *)allones;
+	ipx_broadnet = *(const union ipx_net *)allones;
+	ipx_broadhost = *(const union ipx_host *)allones;
 
 	read_random(&ipx_pexseq, sizeof ipx_pexseq);
 	ipxpcb.ipxp_next = ipxpcb.ipxp_prev = &ipxpcb;
@@ -133,6 +142,9 @@
 	struct ipx_ifaddr *ia;
 	int len;
 
+	/*
+	 * XXXRW: Would be nice to remove this.
+	 */
 	GIANT_REQUIRED;
 
 	/*
--- //depot/vendor/freebsd/src/sys/netipx/ipx_pcb.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/netipx/ipx_pcb.c	2004/02/29 21:00:26
@@ -268,6 +268,7 @@
 {
 	struct socket *so = ipxp->ipxp_socket;
 
+	SOCK_LOCK(so);
 	so->so_pcb = 0;
 	sotryfree(so);
 	if (ipxp->ipxp_route.ro_rt != NULL)
--- //depot/vendor/freebsd/src/sys/netipx/ipx_usrreq.c	2003/11/17 16:40:43
+++ //depot/user/rwatson/netperf/sys/netipx/ipx_usrreq.c	2004/02/28 14:29:37
@@ -423,8 +423,12 @@
 	s = splnet();
 	ipx_pcbdetach(ipxp);
 	splx(s);
-	sotryfree(so);
-	soisdisconnected(so);
+	SOCK_LOCK(so);
+	if (so->so_count != 0) {
+		soisdisconnected(so);
+		SOCK_UNLOCK(so);
+	} else
+		sofree(so);
 	return (0);
 }
 
--- //depot/vendor/freebsd/src/sys/netipx/ipx_var.h	2003/03/04 15:20:46
+++ //depot/user/rwatson/netperf/sys/netipx/ipx_var.h	2004/03/11 19:09:53
@@ -72,8 +72,8 @@
 extern struct sockaddr_ipx ipx_netmask;
 extern struct sockaddr_ipx ipx_hostmask;
 
-extern union ipx_net ipx_zeronet;
-extern union ipx_host ipx_zerohost;
+extern const union ipx_net ipx_zeronet;
+extern const union ipx_host ipx_zerohost;
 extern union ipx_net ipx_broadnet;
 extern union ipx_host ipx_broadhost;
 
--- //depot/vendor/freebsd/src/sys/netipx/spx_usrreq.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/netipx/spx_usrreq.c	2004/03/11 19:09:53
@@ -77,7 +77,7 @@
 #define spxstat spx_istat.newstats
 #endif  
 
-static int spx_backoff[SPX_MAXRXTSHIFT+1] =
+static const int spx_backoff[SPX_MAXRXTSHIFT+1] =
     { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
 
 static	struct spxpcb *spx_close(struct spxpcb *cb);
--- //depot/vendor/freebsd/src/sys/netnatm/natm.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/netnatm/natm.c	2004/02/29 21:00:26
@@ -135,6 +135,7 @@
      * we turn on 'drain' *before* we sofree.
      */
     npcb_free(npcb, NPCB_DESTROY);	/* drain */
+    SOCK_LOCK(so);
     so->so_pcb = NULL;
     sotryfree(so);
  out:
@@ -463,6 +464,7 @@
        */
 
       npcb_free(npcb, NPCB_DESTROY);	/* drain */
+      SOCK_LOCK(so);
       so->so_pcb = NULL;
       sotryfree(so);
 
--- //depot/vendor/freebsd/src/sys/nfsclient/bootp_subr.c	2004/03/12 12:39:01
+++ //depot/user/rwatson/netperf/sys/nfsclient/bootp_subr.c	2004/03/12 21:34:00
@@ -591,7 +591,7 @@
 	int retry;
 	const char *s;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	/*
 	 * Create socket and set its recieve timeout.
@@ -983,7 +983,7 @@
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	error = socreate(AF_INET, &ifctx->so, SOCK_DGRAM, 0, td->td_ucred, td);
 	if (error != 0)
--- //depot/vendor/freebsd/src/sys/nfsclient/krpc_subr.c	2003/11/14 16:30:34
+++ //depot/user/rwatson/netperf/sys/nfsclient/krpc_subr.c	2004/02/28 14:29:37
@@ -215,8 +215,6 @@
 	nam = mhead = NULL;
 	from = NULL;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
-
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
--- //depot/vendor/freebsd/src/sys/nfsclient/nfs_socket.c	2003/11/14 12:56:53
+++ //depot/user/rwatson/netperf/sys/nfsclient/nfs_socket.c	2004/03/01 17:07:22
@@ -160,12 +160,12 @@
 nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 {
 	struct socket *so;
-	int s, error, rcvreserve, sndreserve;
+	int error, rcvreserve, sndreserve;
 	int pktscale;
 	struct sockaddr *saddr;
 	struct thread *td = &thread0; /* only used for socreate and sobind */
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	nmp->nm_so = NULL;
 	saddr = nmp->nm_nam;
@@ -245,25 +245,25 @@
 		 * connect system call but with the wait timing out so
 		 * that interruptible mounts don't hang here for a long time.
 		 */
-		s = splnet();
+		SOCK_LOCK(so);
 		while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-			(void) tsleep(&so->so_timeo,
+			(void) msleep(&so->so_timeo, SOCK_MTX(so),
 			    PSOCK, "nfscon", 2 * hz);
 			if ((so->so_state & SS_ISCONNECTING) &&
 			    so->so_error == 0 && rep &&
 			    (error = nfs_sigintr(nmp, rep, rep->r_td)) != 0) {
 				so->so_state &= ~SS_ISCONNECTING;
-				splx(s);
+				SOCK_UNLOCK(so);
 				goto bad;
 			}
 		}
 		if (so->so_error) {
 			error = so->so_error;
 			so->so_error = 0;
-			splx(s);
+			SOCK_UNLOCK(so);
 			goto bad;
 		}
-		splx(s);
+		SOCK_UNLOCK(so);
 	}
 	so->so_rcv.sb_timeo = 5 * hz;
 	so->so_snd.sb_timeo = 5 * hz;
@@ -383,7 +383,7 @@
 {
 	struct socket *so;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	if (nmp->nm_so) {
 		so = nmp->nm_so;
@@ -419,7 +419,7 @@
 	struct sockaddr *sendnam;
 	int error, soflags, flags;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	KASSERT(rep, ("nfs_send: called with rep == NULL"));
 
@@ -494,7 +494,7 @@
 	int error, sotype, rcvflg;
 	struct thread *td = curthread;	/* XXX */
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	/*
 	 * Set up arguments for soreceive()
@@ -1139,6 +1139,7 @@
 		 * Set r_rtt to -1 in case we fail to send it now.
 		 */
 		rep->r_rtt = -1;
+		SOCKBUF_LOCK(&so->so_snd);
 		if (sbspace(&so->so_snd) >= rep->r_mreq->m_pkthdr.len &&
 		   ((nmp->nm_flag & NFSMNT_DUMBTIMR) ||
 		    (rep->r_flags & R_SENT) ||
@@ -1174,6 +1175,7 @@
 				rep->r_rtt = 0;
 			}
 		}
+		SOCKBUF_UNLOCK(&so->so_snd);
 	}
 	splx(s);
 	nfs_timer_handle = timeout(nfs_timer, NULL, nfs_ticks);
--- //depot/vendor/freebsd/src/sys/nfsclient/nfs_vfsops.c	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/nfsclient/nfs_vfsops.c	2004/03/01 17:07:22
@@ -388,7 +388,7 @@
 	u_long l;
 	char buf[128];
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 #if defined(BOOTP_NFSROOT) && defined(BOOTP)
 	bootpc_init();		/* use bootp to get nfs_diskless filled in */
--- //depot/vendor/freebsd/src/sys/nfsserver/nfs_srvsock.c	2004/03/07 08:30:37
+++ //depot/user/rwatson/netperf/sys/nfsserver/nfs_srvsock.c	2004/03/07 09:40:54
@@ -425,7 +425,7 @@
 		goto dorecs;
 	}
 #endif
-	GIANT_REQUIRED;		/* XXX until socket locking is done */
+	NET_ASSERT_GIANT();
 
 	auio.uio_td = NULL;
 	if (so->so_type == SOCK_STREAM) {
@@ -727,7 +727,7 @@
 	struct sockaddr *sendnam;
 	int error, soflags, flags;
 
-	GIANT_REQUIRED;		/* XXX until socket locking is done */
+	NET_ASSERT_GIANT();
 
 	soflags = so->so_proto->pr_flags;
 	if ((soflags & PR_CONNREQUIRED) || (so->so_state & SS_ISCONNECTED))
--- //depot/vendor/freebsd/src/sys/nfsserver/nfs_syscalls.c	2003/11/07 15:00:50
+++ //depot/user/rwatson/netperf/sys/nfsserver/nfs_syscalls.c	2004/03/01 17:07:22
@@ -199,7 +199,7 @@
 	struct socket *so;
 	int error, s;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	so = fp->f_data;
 #if 0
--- //depot/vendor/freebsd/src/sys/rpc/rpcclnt.c	2004/03/07 08:25:23
+++ //depot/user/rwatson/netperf/sys/rpc/rpcclnt.c	2004/03/07 09:40:54
@@ -362,7 +362,7 @@
 		RPC_RETURN(EFAULT);
 	}
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	/* create the socket */
 	rpc->rc_so = NULL;
@@ -620,7 +620,7 @@
 {
 	struct socket  *so;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	if (rpc->rc_so) {
 		so = rpc->rc_so;
@@ -671,7 +671,7 @@
 #endif
 	int error, soflags, flags;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	if (rep) {
 		if (rep->r_flags & R_SOFTTERM) {
@@ -756,7 +756,7 @@
 #endif
 	int error, sotype, rcvflg;
 
-	GIANT_REQUIRED;		/* XXX until socket locking done */
+	NET_ASSERT_GIANT();
 
 	/*
 	 * Set up arguments for soreceive()
@@ -1436,6 +1436,7 @@
 		 * Set r_rtt to -1 in case we fail to send it now.
 		 */
 		rep->r_rtt = -1;
+		SOCKBUF_LOCK(&so->so_snd);
 		if (sbspace(&so->so_snd) >= rep->r_mreq->m_pkthdr.len &&
 		    ((rpc->rc_flag & RPCCLNT_DUMBTIMR) ||
 		     (rep->r_flags & R_SENT) ||
@@ -1470,6 +1471,7 @@
 				rep->r_rtt = 0;
 			}
 		}
+		SOCKBUF_UNLOCK(&so->so_snd);
 	}
 	splx(s);
 
--- //depot/vendor/freebsd/src/sys/sys/mutex.h	2004/03/01 14:38:06
+++ //depot/user/rwatson/netperf/sys/sys/mutex.h	2004/03/05 07:52:54
@@ -350,19 +350,22 @@
  * without special recursion handling.
  *
  * This mechanism is intended as temporary until everything of
- * importance is properly locked.
+ * importance is properly locked.  Note: the semantics for
+ * NET_{LOCK,UNLOCK}_GIANT() are not the same as DROP_GIANT()
+ * and PICKUP_GIANT(), as they are plain mutex operations
+ * without a recursion counter.
  */
 extern	int debug_mpsafenet;		/* defined in net/netisr.c */
 #define	NET_LOCK_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_lock(&Giant);					\
 } while (0)
 #define	NET_UNLOCK_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_unlock(&Giant);					\
 } while (0)
 #define	NET_ASSERT_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_assert(&Giant, MA_OWNED);				\
 } while (0)
 
--- //depot/vendor/freebsd/src/sys/sys/socketvar.h	2004/02/29 19:15:33
+++ //depot/user/rwatson/netperf/sys/sys/socketvar.h	2004/02/29 21:00:26
@@ -39,6 +39,8 @@
 
 #include <sys/queue.h>			/* for TAILQ macros */
 #include <sys/selinfo.h>		/* for struct selinfo */
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
 
 /*
  * Kernel structure per socket.
@@ -48,19 +50,6 @@
  */
 typedef	u_quad_t so_gen_t;
 
-/*
- * List of locks:
- * (c)	const, inited in either socreate() or sonewconn()
- * (m)	sb_mtx mutex
- * (mr)	so_rcv.sb_mtx mutex
- * (sg)	sigio_lock sx
- * (sh)	sohead_lock sx
- *
- * Lock of so_rcv.sb_mtx can duplicate, provided that sohead_lock
- * is exclusively locked.
- *
- * Brackets mean that this data is not protected yet.
- */
 struct socket {
 	int	so_count;		/* reference count */
 	short	so_type;		/* generic type, see socket.h */
@@ -99,6 +88,7 @@
  */
 	struct sockbuf {
 		struct	selinfo sb_sel;	/* process selecting read/write */
+		struct	mtx sb_mtx;	/* sockbuf lock */
 #define	sb_startzero	sb_mb
 		struct	mbuf *sb_mb;	/* the mbuf chain */
 		struct	mbuf *sb_mbtail; /* the last mbuf in the chain */
@@ -146,6 +136,22 @@
 	}								\
 } while (/*CONSTCOND*/0)
 
+#define	SOCKBUF_MTX(_sb)		(&(_sb)->sb_mtx)
+#define	SOCKBUF_LOCK_INIT(_sb, _name) \
+	mtx_init(SOCKBUF_MTX(_sb), _name, NULL, MTX_DEF)
+#define	SOCKBUF_LOCK_DESTROY(_sb)	mtx_destroy(SOCKBUF_MTX(_sb))
+#define	SOCKBUF_LOCK(_sb)		mtx_lock(SOCKBUF_MTX(_sb))
+#define	SOCKBUF_OWNED(_sb)		mtx_owned(SOCKBUF_MTX(_sb))
+#define	SOCKBUF_UNLOCK(_sb)		mtx_unlock(SOCKBUF_MTX(_sb))
+#define	SOCKBUF_LOCK_ASSERT(_sb)	mtx_assert(SOCKBUF_MTX(_sb), MA_OWNED)
+
+/* NB: we reuse the rcv sockbuf's lock for other items in the socket */
+#define	SOCK_MTX(_so)			SOCKBUF_MTX(&(_so)->so_rcv)
+#define	SOCK_LOCK(_so)			SOCKBUF_LOCK(&(_so)->so_rcv)
+#define	SOCK_OWNED(_so)			SOCKBUF_OWNED(&(_so)->so_rcv)
+#define	SOCK_UNLOCK(_so)		SOCKBUF_UNLOCK(&(_so)->so_rcv)
+#define	SOCK_LOCK_ASSERT(_so)		SOCKBUF_LOCK_ASSERT(&(_so)->so_rcv)
+
 /*
  * Socket state bits.
  */
@@ -197,6 +203,7 @@
 	uid_t	so_uid;		/* XXX */
 };
 
+#ifdef _KERNEL
 /*
  * Macros for sockets and socket buffering.
  */
@@ -267,47 +274,61 @@
 		((sb)->sb_flags |= SB_LOCK), 0)
 
 /* release lock on sockbuf sb */
-#define	sbunlock(sb) { \
+#define	sbunlock(sb) do { \
 	(sb)->sb_flags &= ~SB_LOCK; \
 	if ((sb)->sb_flags & SB_WANT) { \
 		(sb)->sb_flags &= ~SB_WANT; \
 		wakeup(&(sb)->sb_flags); \
 	} \
-}
+} while (0)
 
 /*
  * soref()/sorele() ref-count the socket structure.  Note that you must
  * still explicitly close the socket, but the last ref count will free
  * the structure.
  */
-#define soref(so)	do {			\
-				++(so)->so_count; \
-			} while (0)
+#define soref(so) do {					\
+	++(so)->so_count;				\
+} while (0)
 
-#define sorele(so)	do {				\
-				if ((so)->so_count <= 0)	\
-					panic("sorele");\
-				if (--(so)->so_count == 0)\
-					sofree(so);	\
-			} while (0)
+#define sorele(so) do {					\
+	SOCK_LOCK_ASSERT(so);				\
+	if ((so)->so_count <= 0)			\
+		panic("sorele");			\
+	if (--(so)->so_count == 0)			\
+		sofree(so);				\
+	else						\
+		SOCK_UNLOCK(so);			\
+	so = NULL;					\
+} while (0)
 
-#define sotryfree(so)	do {				\
-				if ((so)->so_count == 0)	\
-					sofree(so);	\
-			} while(0)
+#define sotryfree(so) do {				\
+	SOCK_LOCK_ASSERT(so);				\
+	if ((so)->so_count == 0)			\
+		sofree(so);				\
+	else						\
+		SOCK_UNLOCK(so);			\
+	so = NULL;					\
+} while (0)
 
-#define	sorwakeup(so)	do {					\
-				if (sb_notify(&(so)->so_rcv))	\
-					sowakeup((so), &(so)->so_rcv); \
-			} while (0)
+#define	sorwakeup(so) do {				\
+	if (sb_notify(&(so)->so_rcv))			\
+		sowakeup((so), &(so)->so_rcv);		\
+} while (0)
+#define	sorwakeup_locked(so) do {			\
+	if (sb_notify(&(so)->so_rcv))			\
+		sowakeup_locked((so), &(so)->so_rcv);	\
+} while (0)
 
-#define	sowwakeup(so)	do {					\
-				if (sb_notify(&(so)->so_snd))	\
-					sowakeup((so), &(so)->so_snd); \
-			} while (0)
+#define	sowwakeup(so) do {				\
+	if (sb_notify(&(so)->so_snd))			\
+		sowakeup((so), &(so)->so_snd);		\
+} while (0)
+#define	sowwakeup_locked(so)	do {			\
+	if (sb_notify(&(so)->so_snd))			\
+		sowakeup_locked((so), &(so)->so_snd);	\
+} while (0)
 
-#ifdef _KERNEL
-
 /*
  * Argument structure for sosetopt et seq.  This is in the KERNEL
  * section because it will never be visible to user code.
@@ -355,12 +376,19 @@
 int	sockargs(struct mbuf **mp, caddr_t buf, int buflen, int type);
 int	getsockaddr(struct sockaddr **namp, caddr_t uaddr, size_t len);
 void	sbappend(struct sockbuf *sb, struct mbuf *m);
+void	sbappend_locked(struct sockbuf *sb, struct mbuf *m);
 void	sbappendstream(struct sockbuf *sb, struct mbuf *m);
+void	sbappendstream_locked(struct sockbuf *sb, struct mbuf *m);
 int	sbappendaddr(struct sockbuf *sb, struct sockaddr *asa,
 	    struct mbuf *m0, struct mbuf *control);
+int	sbappendaddr_locked(struct sockbuf *sb, struct sockaddr *asa,
+	    struct mbuf *m0, struct mbuf *control);
 int	sbappendcontrol(struct sockbuf *sb, struct mbuf *m0,
 	    struct mbuf *control);
+int	sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0,
+	    struct mbuf *control);
 void	sbappendrecord(struct sockbuf *sb, struct mbuf *m0);
+void	sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0);
 void	sbcheck(struct sockbuf *sb);
 void	sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n);
 struct mbuf *
@@ -369,6 +397,7 @@
 void	sbdroprecord(struct sockbuf *sb);
 void	sbflush(struct sockbuf *sb);
 void	sbinsertoob(struct sockbuf *sb, struct mbuf *m0);
+void	sbinsertoob_locked(struct sockbuf *sb, struct mbuf *m0);
 void	sbrelease(struct sockbuf *sb, struct socket *so);
 int	sbreserve(struct sockbuf *sb, u_long cc, struct socket *so,
 	    struct thread *td);
@@ -381,7 +410,9 @@
 int	socheckuid(struct socket *so, uid_t uid);
 int	sobind(struct socket *so, struct sockaddr *nam, struct thread *td);
 void	socantrcvmore(struct socket *so);
+void	socantrcvmore_locked(struct socket *so);
 void	socantsendmore(struct socket *so);
+void	socantsendmore_locked(struct socket *so);
 int	soclose(struct socket *so);
 int	soconnect(struct socket *so, struct sockaddr *nam, struct thread *td);
 int	soconnect2(struct socket *so1, struct socket *so2);
@@ -422,6 +453,7 @@
 int	soshutdown(struct socket *so, int how);
 void	sotoxsocket(struct socket *so, struct xsocket *xso);
 void	sowakeup(struct socket *so, struct sockbuf *sb);
+void	sowakeup_locked(struct socket *so, struct sockbuf *sb);
 
 #ifdef SOCKBUF_DEBUG
 void	sblastrecordchk(struct sockbuf *, const char *, int);
--- //depot/vendor/freebsd/src/sys/sys/unpcb.h	2003/02/05 17:35:25
+++ //depot/user/rwatson/netperf/sys/sys/unpcb.h	2004/02/28 14:29:37
@@ -82,6 +82,7 @@
 	unp_gen_t unp_gencnt;		/* generation count of this instance */
 	int	unp_flags;		/* flags */
 	struct	xucred unp_peercred;	/* peer credentials, if applicable */
+	struct	mtx unp_mtx;		/* locking */
 };
 
 /*
