--- //depot/user/rwatson/sockref/src/sys/fs/fifofs/fifo_vnops.c	2006/03/16 21:44:47
+++ //depot/user/rwatson/resock/src/sys/fs/fifofs/fifo_vnops.c	2006/06/18 18:46:27
@@ -203,9 +203,9 @@
 		}
 		fip->fi_readers = fip->fi_writers = 0;
 		wso->so_snd.sb_lowat = PIPE_BUF;
-		SOCKBUF_LOCK(&rso->so_rcv);
+		SOCK_LOCK(rso);
 		rso->so_rcv.sb_state |= SBS_CANTRCVMORE;
-		SOCKBUF_UNLOCK(&rso->so_rcv);
+		SOCK_UNLOCK(rso);
 		KASSERT(vp->v_fifoinfo == NULL,
 		    ("fifo_open: v_fifoinfo race"));
 		vp->v_fifoinfo = fip;
@@ -225,9 +225,12 @@
 	if (ap->a_mode & FREAD) {
 		fip->fi_readers++;
 		if (fip->fi_readers == 1) {
-			SOCKBUF_LOCK(&fip->fi_writesock->so_snd);
+			/*
+			 * XXXRW: Coalesce locking to include sowwakeip()?
+			 */
+			SOCK_LOCK(fip->fi_writesock);
 			fip->fi_writesock->so_snd.sb_state &= ~SBS_CANTSENDMORE;
-			SOCKBUF_UNLOCK(&fip->fi_writesock->so_snd);
+			SOCK_UNLOCK(fip->fi_writesock);
 			if (fip->fi_writers > 0) {
 				wakeup(&fip->fi_writers);
 				sowwakeup(fip->fi_writesock);
@@ -241,9 +244,12 @@
 		}
 		fip->fi_writers++;
 		if (fip->fi_writers == 1) {
-			SOCKBUF_LOCK(&fip->fi_readsock->so_rcv);
+			/*
+			 * XXXRW: Coalesce locking to include sowwakeip()?
+			 */
+			SOCK_LOCK(fip->fi_readsock);
 			fip->fi_readsock->so_rcv.sb_state &= ~SBS_CANTRCVMORE;
-			SOCKBUF_UNLOCK(&fip->fi_readsock->so_rcv);
+			SOCK_UNLOCK(fip->fi_readsock);
 			if (fip->fi_readers > 0) {
 				wakeup(&fip->fi_readers);
 				sorwakeup(fip->fi_readsock);
@@ -342,11 +348,11 @@
 {
 	struct socket *so = (struct socket *)kn->kn_hook;
 
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	knlist_remove(&so->so_rcv.sb_sel.si_note, kn, 1);
 	if (knlist_empty(&so->so_rcv.sb_sel.si_note))
 		so->so_rcv.sb_flags &= ~SB_KNOTE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 }
 
 static int
@@ -354,7 +360,7 @@
 {
 	struct socket *so = (struct socket *)kn->kn_hook;
 
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	kn->kn_data = so->so_rcv.sb_cc;
 	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 		kn->kn_flags |= EV_EOF;
@@ -370,11 +376,11 @@
 {
 	struct socket *so = (struct socket *)kn->kn_hook;
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	knlist_remove(&so->so_snd.sb_sel.si_note, kn, 1);
 	if (knlist_empty(&so->so_snd.sb_sel.si_note))
 		so->so_snd.sb_flags &= ~SB_KNOTE;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 }
 
 static int
@@ -382,7 +388,7 @@
 {
 	struct socket *so = (struct socket *)kn->kn_hook;
 
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
 	kn->kn_data = sbspace(&so->so_snd);
 	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
@@ -646,10 +652,10 @@
 
 	kn->kn_hook = (caddr_t)so;
 
-	SOCKBUF_LOCK(sb);
+	SOCK_LOCK(so);
 	knlist_add(&sb->sb_sel.si_note, kn, 1);
 	sb->sb_flags |= SB_KNOTE;
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 
 	return (0);
 }
--- //depot/user/rwatson/sockref/src/sys/fs/portalfs/portal_vnops.c	2006/07/19 14:21:12
+++ //depot/user/rwatson/resock/src/sys/fs/portalfs/portal_vnops.c	2006/07/22 17:07:41
@@ -296,15 +296,12 @@
 	/*
 	 * Set miscellaneous flags
 	 */
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_rcv.sb_timeo = 0;
 	so->so_rcv.sb_flags |= SB_NOINTR;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_timeo = 0;
 	so->so_snd.sb_flags |= SB_NOINTR;
-	SOCKBUF_UNLOCK(&so->so_snd);
-
+	SOCK_UNLOCK(so);
 
 	pcred.pcr_flag = ap->a_mode;
 	pcred.pcr_uid = ap->a_cred->cr_uid;
--- //depot/user/rwatson/sockref/src/sys/kern/sys_socket.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/kern/sys_socket.c	2006/11/30 15:30:26
@@ -150,32 +150,18 @@
 		break;
 
 	case FIOASYNC:
-		/*
-		 * XXXRW: This code separately acquires SOCK_LOCK(so)
-		 * and SOCKBUF_LOCK(&so->so_rcv) even though they are
-		 * the same mutex to avoid introducing the assumption
-		 * that they are the same.
-		 */
 		if (*(int *)data) {
 			SOCK_LOCK(so);
 			so->so_state |= SS_ASYNC;
-			SOCK_UNLOCK(so);
-			SOCKBUF_LOCK(&so->so_rcv);
 			so->so_rcv.sb_flags |= SB_ASYNC;
-			SOCKBUF_UNLOCK(&so->so_rcv);
-			SOCKBUF_LOCK(&so->so_snd);
 			so->so_snd.sb_flags |= SB_ASYNC;
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 		} else {
 			SOCK_LOCK(so);
 			so->so_state &= ~SS_ASYNC;
-			SOCK_UNLOCK(so);
-			SOCKBUF_LOCK(&so->so_rcv);
 			so->so_rcv.sb_flags &= ~SB_ASYNC;
-			SOCKBUF_UNLOCK(&so->so_rcv);
-			SOCKBUF_LOCK(&so->so_snd);
 			so->so_snd.sb_flags &= ~SB_ASYNC;
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 		}
 		break;
 
@@ -262,11 +248,11 @@
 	bzero((caddr_t)ub, sizeof (*ub));
 	ub->st_mode = S_IFSOCK;
 	NET_LOCK_GIANT();
+	SOCK_LOCK(so);
 #ifdef MAC
-	SOCK_LOCK(so);
 	error = mac_check_socket_stat(active_cred, so);
-	SOCK_UNLOCK(so);
 	if (error) {
+		SOCK_UNLOCK(so);
 		NET_UNLOCK_GIANT();
 		return (error);
 	}
@@ -274,11 +260,7 @@
 	/*
 	 * If SBS_CANTRCVMORE is set, but there's still data left in the
 	 * receive buffer, the socket is still readable.
-	 *
-	 * XXXRW: perhaps should lock socket buffer so st_size result
-	 * is consistent.
 	 */
-	/* Unlocked read. */
 	if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0 ||
 	    so->so_rcv.sb_cc != 0)
 		ub->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
@@ -287,6 +269,7 @@
 	ub->st_size = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
 	ub->st_uid = so->so_cred->cr_uid;
 	ub->st_gid = so->so_cred->cr_gid;
+	SOCK_UNLOCK(so);
 	error = (*so->so_proto->pr_usrreqs->pru_sense)(so, ub);
 	NET_UNLOCK_GIANT();
 	return (error);
--- //depot/user/rwatson/sockref/src/sys/kern/uipc_sockbuf.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/kern/uipc_sockbuf.c	2006/11/30 15:30:26
@@ -81,40 +81,40 @@
 socantsendmore_locked(struct socket *so)
 {
 
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
 
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sowwakeup_locked(so);
-	mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
+	/* XXXRW: Note, used turn return the buffer unlocked. */
 }
 
 void
 socantsendmore(struct socket *so)
 {
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	socantsendmore_locked(so);
-	mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
+	SOCK_UNLOCK(so);
 }
 
 void
 socantrcvmore_locked(struct socket *so)
 {
 
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
 	sorwakeup_locked(so);
-	mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
+	/* XXXRW: Note, used turn return the buffer unlocked. */
 }
 
 void
 socantrcvmore(struct socket *so)
 {
 
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	socantrcvmore_locked(so);
-	mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
+	SOCK_UNLOCK(so);
 }
 
 /*
@@ -127,7 +127,7 @@
 	SOCKBUF_LOCK_ASSERT(sb);
 
 	sb->sb_flags |= SB_WAIT;
-	return (msleep(&sb->sb_cc, &sb->sb_mtx,
+	return (msleep(&sb->sb_cc, SOCKBUF_MTX(sb),
 	    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
 	    sb->sb_timeo));
 }
@@ -145,7 +145,7 @@
 
 	while (sb->sb_flags & SB_LOCK) {
 		sb->sb_flags |= SB_WANT;
-		error = msleep(&sb->sb_flags, &sb->sb_mtx,
+		error = msleep(&sb->sb_flags, SOCKBUF_MTX(sb),
 		    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH,
 		    "sblock", 0);
 		if (error)
@@ -159,19 +159,14 @@
  * Wakeup processes waiting on a socket buffer.  Do asynchronous notification
  * via SIGIO if the socket has the SS_ASYNC flag set.
  *
- * Called with the socket buffer lock held; will release the lock by the end
- * of the function.  This allows the caller to acquire the socket buffer lock
- * while testing for the need for various sorts of wakeup and hold it through
- * to the point where it's no longer required.  We currently hold the lock
- * through calls out to other subsystems (with the exception of kqueue), and
- * then release it to avoid lock order issues.  It's not clear that's
- * correct.
+ * XXXRW: Note: sowakeup() used to return unlocked, now returns locked.  This
+ * is almost certainly an improvement.
  */
 void
 sowakeup(struct socket *so, struct sockbuf *sb)
 {
 
-	SOCKBUF_LOCK_ASSERT(sb);
+	SOCK_LOCK_ASSERT(so);
 
 	selwakeuppri(&sb->sb_sel, PSOCK);
 	sb->sb_flags &= ~SB_SEL;
@@ -180,14 +175,12 @@
 		wakeup(&sb->sb_cc);
 	}
 	KNOTE_LOCKED(&sb->sb_sel.si_note, 0);
-	SOCKBUF_UNLOCK(sb);
 	if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
 		pgsigio(&so->so_sigio, SIGIO, 0);
 	if (sb->sb_flags & SB_UPCALL)
 		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
 	if (sb->sb_flags & SB_AIO)
 		aio_swake(so, sb);
-	mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED);
 }
 
 /*
@@ -220,14 +213,16 @@
  * some of the available buffer space in the system buffer pool for the
  * socket (currently, it does nothing but enforce limits).  The space should
  * be released by calling sbrelease() when the socket is destroyed.
+ *
+ * XXXRW: Would it make sense to assert the lock and let the caller acquire
+ * it?
  */
 int
 soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
 {
 	struct thread *td = curthread;
 
-	SOCKBUF_LOCK(&so->so_snd);
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0)
 		goto bad;
 	if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0)
@@ -238,14 +233,12 @@
 		so->so_snd.sb_lowat = MCLBYTES;
 	if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
 		so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 	return (0);
 bad2:
 	sbrelease_locked(&so->so_snd, so);
 bad:
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 	return (ENOBUFS);
 }
 
@@ -275,7 +268,7 @@
 {
 	rlim_t sbsize_limit;
 
-	SOCKBUF_LOCK_ASSERT(sb);
+	SOCK_LOCK_ASSERT(so);
 
 	/*
 	 * td will only be NULL when we're in an interrupt (e.g. in
@@ -306,9 +299,9 @@
 {
 	int error;
 
-	SOCKBUF_LOCK(sb);
+	SOCK_LOCK(so);
 	error = sbreserve_locked(sb, cc, so, td);
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 	return (error);
 }
 
@@ -329,7 +322,7 @@
 sbrelease_locked(struct sockbuf *sb, struct socket *so)
 {
 
-	SOCKBUF_LOCK_ASSERT(sb);
+	SOCK_LOCK_ASSERT(so);
 
 	sbrelease_internal(sb, so);
 }
@@ -338,9 +331,9 @@
 sbrelease(struct sockbuf *sb, struct socket *so)
 {
 
-	SOCKBUF_LOCK(sb);
+	SOCK_LOCK(so);
 	sbrelease_locked(sb, so);
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 }
 
 void
--- //depot/user/rwatson/sockref/src/sys/kern/uipc_socket.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/kern/uipc_socket.c	2007/04/29 12:41:48
@@ -269,8 +269,7 @@
 		return (NULL);
 	}
 #endif
-	SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd");
-	SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv");
+	SOCK_LOCK_INIT(so);
 	TAILQ_INIT(&so->so_aiojobq);
 	mtx_lock(&so_global_mtx);
 	so->so_gencnt = ++so_gencnt;
@@ -310,8 +309,7 @@
 	mac_destroy_socket(so);
 #endif
 	crfree(so->so_cred);
-	SOCKBUF_LOCK_DESTROY(&so->so_snd);
-	SOCKBUF_LOCK_DESTROY(&so->so_rcv);
+	SOCK_LOCK_DESTROY(so);
 	uma_zfree(socket_zone, so);
 }
 
@@ -362,9 +360,9 @@
 #ifdef MAC
 	mac_create_socket(cred, so);
 #endif
-	knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv),
+	knlist_init(&so->so_rcv.sb_sel.si_note, SOCK_MTX(so),
 	    NULL, NULL, NULL);
-	knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd),
+	knlist_init(&so->so_snd.sb_sel.si_note, SOCK_MTX(so),
 	    NULL, NULL, NULL);
 	so->so_count = 1;
 	error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
@@ -428,10 +426,10 @@
 	mac_create_socket_from_socket(head, so);
 	SOCK_UNLOCK(head);
 #endif
-	knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv),
-	    NULL, NULL, NULL);
-	knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd),
-	    NULL, NULL, NULL);
+	knlist_init(&so->so_rcv.sb_sel.si_note, SOCK_MTX(so), NULL, NULL,
+	    NULL);
+	knlist_init(&so->so_snd.sb_sel.si_note, SOCK_MTX(so), NULL, NULL,
+	    NULL);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
 		sodealloc(so);
@@ -596,10 +594,14 @@
 		KASSERT((TAILQ_EMPTY(&so->so_comp)), ("sofree: so_comp populated"));
 		KASSERT((TAILQ_EMPTY(&so->so_incomp)), ("sofree: so_comp populated"));
 	}
-	SOCK_UNLOCK(so);
 	ACCEPT_UNLOCK();
 
 	/*
+	 * XXXRW: A lot of the work done here is a waste of time.  We just
+	 * need sbdestroy() which drains the mbufs, etc, and doesn't bother
+	 * with wakeups, etc.
+	 */
+	/*
 	 * From this point on, we assume that no other references to this
 	 * socket exist anywhere else in the stack.  Therefore, no locks need
 	 * to be acquired or held.
@@ -689,6 +691,7 @@
 		}
 		ACCEPT_UNLOCK();
 	}
+	/* XXXRW: Can we now hold SOCK_LOCK() over all of the below? */
 	ACCEPT_LOCK();
 	SOCK_LOCK(so);
 	KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF"));
@@ -742,6 +745,10 @@
 {
 	int error;
 
+	/*
+	 * XXXRW: Should this state transition be generated by pru_accept()
+	 * and a call to soaccept_proto()?
+	 */
 	SOCK_LOCK(so);
 	KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF"));
 	so->so_state &= ~SS_NOFDREF;
@@ -758,8 +765,14 @@
 {
 	int error;
 
+	/*
+	 * XXXRW: Should this state transition be driven by the protocol layer
+	 * using a soconnect_proto() so that SOCK_LOCK() can be held over all
+	 * of this?
+	 */
 	if (so->so_options & SO_ACCEPTCONN)
 		return (EOPNOTSUPP);
+
 	/*
 	 * If protocol is connection-based, can only connect once.
 	 * Otherwise, if connected, try to disconnect first.  This allows
@@ -796,6 +809,10 @@
 {
 	int error;
 
+	/*
+	 * XXXRW: Should these checks be done using a sodisconnect_proto() so
+	 * that SOCK_LOCK() can be held over them?
+	 */
 	if ((so->so_state & SS_ISCONNECTED) == 0)
 		return (ENOTCONN);
 	if (so->so_state & SS_ISDISCONNECTING)
@@ -995,16 +1012,16 @@
 	if (control != NULL)
 		clen = control->m_len;
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 		error = EPIPE;
 		goto out;
 	}
 	if (so->so_error) {
 		error = so->so_error;
 		so->so_error = 0;
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 		goto out;
 	}
 	if ((so->so_state & SS_ISCONNECTED) == 0) {
@@ -1017,7 +1034,7 @@
 		    (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
 			if ((so->so_state & SS_ISCONFIRMING) == 0 &&
 			    !(resid == 0 && clen != 0)) {
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				error = ENOTCONN;
 				goto out;
 			}
@@ -1026,7 +1043,7 @@
 				error = ENOTCONN;
 			else
 				error = EDESTADDRREQ;
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			goto out;
 		}
 	}
@@ -1039,7 +1056,7 @@
 	if (flags & MSG_OOB)
 		space += 1024;
 	space -= clen;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	if (resid > space) {
 		error = EMSGSIZE;
 		goto out;
@@ -1166,14 +1183,14 @@
 	if (control != NULL)
 		clen = control->m_len;
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 restart:
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
 	error = sblock(&so->so_snd, SBLOCKWAIT(flags));
 	if (error)
 		goto out_locked;
 	do {
-		SOCKBUF_LOCK_ASSERT(&so->so_snd);
+		SOCK_LOCK_ASSERT(so);
 		if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 			snderr(EPIPE);
 		if (so->so_error) {
@@ -1213,7 +1230,7 @@
 				goto out_locked;
 			goto restart;
 		}
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 		space -= clen;
 		do {
 			if (uio == NULL) {
@@ -1225,7 +1242,7 @@
 				error = sosend_copyin(uio, &top, atomic,
 				    &space, flags);
 				if (error != 0) {
-					SOCKBUF_LOCK(&so->so_snd);
+					SOCK_LOCK(so);
 					goto release;
 				}
 #else
@@ -1234,7 +1251,7 @@
 				    (atomic ? M_PKTHDR : 0) |
 				    ((flags & MSG_EOR) ? M_EOR : 0));
 				if (top == NULL) {
-					SOCKBUF_LOCK(&so->so_snd);
+					SOCK_UNLOCK(so);
 					error = EFAULT; /* only possible error */
 					goto release;
 				}
@@ -1280,19 +1297,19 @@
 			control = NULL;
 			top = NULL;
 			if (error) {
-				SOCKBUF_LOCK(&so->so_snd);
+				SOCK_LOCK(so);
 				goto release;
 			}
 		} while (resid && space > 0);
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_LOCK(so);
 	} while (resid);
 
 release:
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
 	sbunlock(&so->so_snd);
 out_locked:
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
+	SOCK_UNLOCK(so);
 out:
 	if (top != NULL)
 		m_freem(top);
@@ -1386,6 +1403,7 @@
 {
 
 	SOCKBUF_LOCK_ASSERT(sb);
+
 	/*
 	 * First, update for the new value of nextrecord.  If necessary, make
 	 * it the first record.
@@ -1458,9 +1476,9 @@
 	    && uio->uio_resid)
 		(*pr->pr_usrreqs->pru_rcvd)(so, 0);
 
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 restart:
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
 	if (error)
 		goto out;
@@ -1493,7 +1511,7 @@
 				so->so_error = 0;
 			goto release;
 		}
-		SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+		SOCK_LOCK_ASSERT(so);
 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 			if (m)
 				goto dontblock;
@@ -1541,7 +1559,7 @@
 	 * By holding the high-level sblock(), we prevent simultaneous
 	 * readers from pulling off the front of the socket buffer.
 	 */
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	if (uio->uio_td)
 		uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++;
 	KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
@@ -1597,10 +1615,10 @@
 			cmn = cm->m_next;
 			cm->m_next = NULL;
 			if (pr->pr_domain->dom_externalize != NULL) {
-				SOCKBUF_UNLOCK(&so->so_rcv);
+				SOCK_UNLOCK(so);
 				error = (*pr->pr_domain->dom_externalize)
 				    (cm, controlp);
-				SOCKBUF_LOCK(&so->so_rcv);
+				SOCK_LOCK(so);
 			} else if (controlp != NULL)
 				*controlp = cm;
 			else
@@ -1642,7 +1660,7 @@
 			}
 		}
 	}
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	SBLASTRECORDCHK(&so->so_rcv);
 	SBLASTMBUFCHK(&so->so_rcv);
 
@@ -1661,7 +1679,7 @@
 		 * If the type of mbuf has changed since the last mbuf
 		 * examined ('type'), end the receive operation.
 	 	 */
-		SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+		SOCK_LOCK_ASSERT(so);
 		if (m->m_type == MT_OOBDATA) {
 			if (type != MT_OOBDATA)
 				break;
@@ -1684,10 +1702,10 @@
 		 * to the sockbuf when we block interrupts again.
 		 */
 		if (mp == NULL) {
-			SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+			SOCK_LOCK_ASSERT(so);
 			SBLASTRECORDCHK(&so->so_rcv);
 			SBLASTMBUFCHK(&so->so_rcv);
-			SOCKBUF_UNLOCK(&so->so_rcv);
+			SOCK_UNLOCK(so);
 #ifdef ZERO_COPY_SOCKETS
 			if (so_zero_copy_receive) {
 				int disposable;
@@ -1704,7 +1722,7 @@
 			} else
 #endif /* ZERO_COPY_SOCKETS */
 			error = uiomove(mtod(m, char *) + moff, (int)len, uio);
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 			if (error) {
 				/*
 				 * The MT_SONAME mbuf has already been removed
@@ -1722,7 +1740,7 @@
 			}
 		} else
 			uio->uio_resid -= len;
-		SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+		SOCK_LOCK_ASSERT(so);
 		if (len == m->m_len - moff) {
 			if (m->m_flags & M_EOR)
 				flags |= MSG_EOR;
@@ -1757,10 +1775,10 @@
 					else
 						copy_flag = M_TRYWAIT;
 					if (copy_flag == M_TRYWAIT)
-						SOCKBUF_UNLOCK(&so->so_rcv);
+						SOCK_UNLOCK(so);
 					*mp = m_copym(m, 0, len, copy_flag);
 					if (copy_flag == M_TRYWAIT)
-						SOCKBUF_LOCK(&so->so_rcv);
+						SOCK_LOCK(so);
  					if (*mp == NULL) {
  						/*
  						 * m_copym() couldn't
@@ -1779,7 +1797,7 @@
 				so->so_rcv.sb_cc -= len;
 			}
 		}
-		SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+		SOCK_LOCK_ASSERT(so);
 		if (so->so_oobmark) {
 			if ((flags & MSG_PEEK) == 0) {
 				so->so_oobmark -= len;
@@ -1804,7 +1822,7 @@
 		 */
 		while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
 		    !sosendallatonce(so) && nextrecord == NULL) {
-			SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+			SOCK_LOCK_ASSERT(so);
 			if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				break;
 			/*
@@ -1812,9 +1830,9 @@
 			 * drained before blocking.
 			 */
 			if (pr->pr_flags & PR_WANTRCVD) {
-				SOCKBUF_UNLOCK(&so->so_rcv);
+				SOCK_UNLOCK(so);
 				(*pr->pr_usrreqs->pru_rcvd)(so, flags);
-				SOCKBUF_LOCK(&so->so_rcv);
+				SOCK_LOCK(so);
 			}
 			SBLASTRECORDCHK(&so->so_rcv);
 			SBLASTMBUFCHK(&so->so_rcv);
@@ -1827,7 +1845,7 @@
 		}
 	}
 
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	if (m != NULL && pr->pr_flags & PR_ATOMIC) {
 		flags |= MSG_TRUNC;
 		if ((flags & MSG_PEEK) == 0)
@@ -1856,12 +1874,12 @@
 		 */
 		if (!(flags & MSG_SOCALLBCK) &&
 		    (pr->pr_flags & PR_WANTRCVD)) {
-			SOCKBUF_UNLOCK(&so->so_rcv);
+			SOCK_UNLOCK(so);
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 		}
 	}
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	if (orig_resid == uio->uio_resid && orig_resid &&
 	    (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
 		sbunlock(&so->so_rcv);
@@ -1871,11 +1889,11 @@
 	if (flagsp != NULL)
 		*flagsp |= flags;
 release:
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 	sbunlock(&so->so_rcv);
 out:
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
+	SOCK_UNLOCK(so);
 	return (error);
 }
 
@@ -1915,31 +1933,26 @@
 }
 
 void
-sorflush(so)
+sorflush_locked(so)
 	struct socket *so;
 {
 	struct sockbuf *sb = &so->so_rcv;
 	struct protosw *pr = so->so_proto;
 	struct sockbuf asb;
 
+	SOCK_LOCK_ASSERT(so);
+
 	/*
-	 * XXXRW: This is quite ugly.  Previously, this code made a copy of
-	 * the socket buffer, then zero'd the original to clear the buffer
-	 * fields.  However, with mutexes in the socket buffer, this causes
-	 * problems.  We only clear the zeroable bits of the original;
-	 * however, we have to initialize and destroy the mutex in the copy
-	 * so that dom_dispose() and sbrelease() can lock t as needed.
+	 * XXXRW: This is quite ugly -- we create a stack-local socket buffer
+	 * so that we can release the lock on the socket's copy of the buffer,
+	 * as dom_dispose() may block (among other things).  It's a bit
+	 * cleaner with a single socket mutex than with socket buffer mutexes,
+	 * as we don't have to mock up a socket buffer mutex for the benefit
+	 * of sbrelease().
 	 */
-	SOCKBUF_LOCK(sb);
 	sb->sb_flags |= SB_NOINTR;
 	(void) sblock(sb, M_WAITOK);
-	/*
-	 * socantrcvmore_locked() drops the socket buffer mutex so that it
-	 * can safely perform wakeups.  Re-acquire the mutex before
-	 * continuing.
-	 */
 	socantrcvmore_locked(so);
-	SOCKBUF_LOCK(sb);
 	sbunlock(sb);
 	/*
 	 * Invalidate/clear most of the sockbuf structure, but leave selinfo
@@ -1950,13 +1963,22 @@
 	    sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
 	bzero(&sb->sb_startzero,
 	    sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 
-	SOCKBUF_LOCK_INIT(&asb, "so_rcv");
+	asb.sb_somtxp = NULL;
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
 	sbrelease(&asb, so);
-	SOCKBUF_LOCK_DESTROY(&asb);
+}
+
+void
+sorflush(so)
+	struct socket *so;
+{
+
+	SOCK_LOCK(so);
+	sorflush_locked(so);
+	SOCK_UNLOCK_ASSERT(so);
 }
 
 /*
@@ -2112,18 +2134,18 @@
 			 * high-water.
 			 */
 			case SO_SNDLOWAT:
-				SOCKBUF_LOCK(&so->so_snd);
+				SOCK_LOCK(so);
 				so->so_snd.sb_lowat =
 				    (optval > so->so_snd.sb_hiwat) ?
 				    so->so_snd.sb_hiwat : optval;
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				break;
 			case SO_RCVLOWAT:
-				SOCKBUF_LOCK(&so->so_rcv);
+				SOCK_LOCK(so);
 				so->so_rcv.sb_lowat =
 				    (optval > so->so_rcv.sb_hiwat) ?
 				    so->so_rcv.sb_hiwat : optval;
-				SOCKBUF_UNLOCK(&so->so_rcv);
+				SOCK_UNLOCK(so);
 				break;
 			}
 			break;
@@ -2280,6 +2302,7 @@
 			break;
 
 		case SO_TYPE:
+			/* Lockless read. */
 			optval = so->so_type;
 			goto integer;
 
@@ -2291,23 +2314,28 @@
 			goto integer;
 
 		case SO_SNDBUF:
+			/* Lockless read. */
 			optval = so->so_snd.sb_hiwat;
 			goto integer;
 
 		case SO_RCVBUF:
+			/* Lockless read. */
 			optval = so->so_rcv.sb_hiwat;
 			goto integer;
 
 		case SO_SNDLOWAT:
+			/* Lockless read. */
 			optval = so->so_snd.sb_lowat;
 			goto integer;
 
 		case SO_RCVLOWAT:
+			/* Lockless read. */
 			optval = so->so_rcv.sb_lowat;
 			goto integer;
 
 		case SO_SNDTIMEO:
 		case SO_RCVTIMEO:
+			/* Lockless read. */
 			optval = (sopt->sopt_name == SO_SNDTIMEO ?
 				  so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
 
@@ -2522,8 +2550,7 @@
 {
 	int revents = 0;
 
-	SOCKBUF_LOCK(&so->so_snd);
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	if (events & (POLLIN | POLLRDNORM))
 		if (soreadable(so))
 			revents |= events & (POLLIN | POLLRDNORM);
@@ -2554,9 +2581,8 @@
 			so->so_snd.sb_flags |= SB_SEL;
 		}
 	}
+	SOCK_UNLOCK(so);
 
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_UNLOCK(&so->so_snd);
 	return (revents);
 }
 
@@ -2582,10 +2608,10 @@
 		return (EINVAL);
 	}
 
-	SOCKBUF_LOCK(sb);
+	SOCK_LOCK(so);
 	knlist_add(&sb->sb_sel.si_note, kn, 1);
 	sb->sb_flags |= SB_KNOTE;
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 	return (0);
 }
 
@@ -2594,11 +2620,11 @@
 {
 	struct socket *so = kn->kn_fp->f_data;
 
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	knlist_remove(&so->so_rcv.sb_sel.si_note, kn, 1);
 	if (knlist_empty(&so->so_rcv.sb_sel.si_note))
 		so->so_rcv.sb_flags &= ~SB_KNOTE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 }
 
 /*ARGSUSED*/
@@ -2608,7 +2634,7 @@
 	struct socket *so;
 
 	so = kn->kn_fp->f_data;
-	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
+	SOCK_LOCK_ASSERT(so);
 
 	kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
 	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
@@ -2628,11 +2654,11 @@
 {
 	struct socket *so = kn->kn_fp->f_data;
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	knlist_remove(&so->so_snd.sb_sel.si_note, kn, 1);
 	if (knlist_empty(&so->so_snd.sb_sel.si_note))
 		so->so_snd.sb_flags &= ~SB_KNOTE;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 }
 
 /*ARGSUSED*/
@@ -2642,7 +2668,8 @@
 	struct socket *so;
 
 	so = kn->kn_fp->f_data;
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
+
 	kn->kn_data = sbspace(&so->so_snd);
 	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
--- //depot/user/rwatson/sockref/src/sys/kern/uipc_socket2.c	2006/08/06 19:49:25
+++ //depot/user/rwatson/resock/src/sys/kern/uipc_socket2.c	2006/11/30 15:30:26
@@ -146,16 +146,18 @@
 	/*
 	 * Note: This code assumes that SOCK_LOCK(so) and
 	 * SOCKBUF_LOCK(&so->so_rcv) are the same.
+	 *
+	 * XXXRW: Set all flags before any upcalls?
 	 */
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_state &= ~SS_ISCONNECTING;
 	so->so_state |= SS_ISDISCONNECTING;
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
 	sorwakeup_locked(so);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
+	SOCK_UNLOCK(so);
 }
 
 void
@@ -166,17 +168,19 @@
 	/*
 	 * Note: This code assumes that SOCK_LOCK(so) and
 	 * SOCKBUF_LOCK(&so->so_rcv) are the same.
+	 *
+	 * XXXRW: Set all flags before any upcalls?
 	 */
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISDISCONNECTED;
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
 	sorwakeup_locked(so);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
 	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
+	SOCK_UNLOCK(so);
 }
 
 /*
@@ -361,6 +365,9 @@
 void
 sotoxsocket(struct socket *so, struct xsocket *xso)
 {
+
+	SOCK_LOCK_ASSERT(so);
+
 	xso->xso_len = sizeof *xso;
 	xso->xso_so = so;
 	xso->so_type = so->so_type;
@@ -391,6 +398,9 @@
 void
 sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
 {
+
+	SOCKBUF_LOCK_ASSERT(sb);
+
 	xsb->sb_cc = sb->sb_cc;
 	xsb->sb_hiwat = sb->sb_hiwat;
 	xsb->sb_mbcnt = sb->sb_mbcnt;
--- //depot/user/rwatson/sockref/src/sys/kern/uipc_syscalls.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/kern/uipc_syscalls.c	2006/12/01 12:26:48
@@ -293,6 +293,10 @@
 /*
  * accept1()
  * MPSAFE
+ *
+ * XXXRW: Would it make sense to hold both the accept and socket mutex for the
+ * 'head' socket here, in order to avoid repeated lockless reads of fields
+ * like so_error?
  */
 static int
 accept1(td, uap, compat)
@@ -387,12 +391,15 @@
 	if (error)
 		goto done;
 	ACCEPT_LOCK();
+	/* Lockless read. */
 	if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->so_comp)) {
 		ACCEPT_UNLOCK();
 		error = EWOULDBLOCK;
 		goto noconnection;
 	}
+	/* Lockless read. */
 	while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
+		/* Lockless read. */
 		if (head->so_rcv.sb_state & SBS_CANTRCVMORE) {
 			head->so_error = ECONNABORTED;
 			break;
@@ -404,6 +411,7 @@
 			goto noconnection;
 		}
 	}
+	/* Lockless read. */
 	if (head->so_error) {
 		error = head->so_error;
 		head->so_error = 0;
@@ -581,18 +589,19 @@
 #ifdef MAC
 	SOCK_LOCK(so);
 	error = mac_check_socket_connect(td->td_ucred, so, sa);
-	SOCK_UNLOCK(so);
 	if (error)
 		goto bad;
+	SOCK_UNLOCK(so);
 #endif
 	error = soconnect(so, sa, td);
+	SOCK_LOCK(so);
 	if (error)
 		goto bad;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
+		SOCK_UNLOCK(so);
 		error = EINPROGRESS;
 		goto done1;
 	}
-	SOCK_LOCK(so);
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
 		error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH,
 		    "connec", 0);
@@ -606,10 +615,10 @@
 		error = so->so_error;
 		so->so_error = 0;
 	}
-	SOCK_UNLOCK(so);
 bad:
 	if (!interrupted)
 		so->so_state &= ~SS_ISCONNECTING;
+	SOCK_UNLOCK(so);
 	if (error == ERESTART)
 		error = EINTR;
 done1:
@@ -1656,6 +1665,7 @@
 	if (error)
 		goto done2;
 	so = fp->f_data;
+	/* XXXRW: Lockless read -- check at wrong layer? */
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
 		error = ENOTCONN;
 		goto done1;
@@ -1992,9 +2002,9 @@
 	}
 
 	/* Protect against multiple writers to the socket. */
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	(void) sblock(&so->so_snd, M_WAITOK);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 
 	/*
 	 * Loop through the pages of the file, starting with the requested
@@ -2028,18 +2038,18 @@
 		 * we were not careful here we would send off only one
 		 * sfbuf at a time.
 		 */
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_LOCK(so);
 		if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2)
 			so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2;
 retry_space:
 		if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 			error = EPIPE;
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			goto done;
 		} else if (so->so_error) {
 			error = so->so_error;
 			so->so_error = 0;
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			goto done;
 		}
 		space = sbspace(&so->so_snd);
@@ -2047,7 +2057,7 @@
 		    (space <= 0 ||
 		     space < so->so_snd.sb_lowat)) {
 			if (so->so_state & SS_NBIO) {
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				error = EAGAIN;
 				goto done;
 			}
@@ -2064,12 +2074,12 @@
 			 * then return bytes sent, otherwise return the error.
 			 */
 			if (error) {
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				goto done;
 			}
 			goto retry_space;
 		}
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 
 		/*
 		 * Loop and construct maximum sized mbuf chain to be bulk
@@ -2258,13 +2268,13 @@
 			int mlen;
 
 			mlen = m_length(m, NULL);
-			SOCKBUF_LOCK(&so->so_snd);
+			SOCK_LOCK(so);
 			if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 				error = EPIPE;
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				goto done;
 			}
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			error = (*so->so_proto->pr_usrreqs->pru_send)
 				    (so, 0, m, NULL, NULL, td);
 			if (!error)
@@ -2288,9 +2298,9 @@
 	}
 
 done:
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	sbunlock(&so->so_snd);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 out:
 	/*
 	 * If there was no error we have to clear td->td_retval[0]
--- //depot/user/rwatson/sockref/src/sys/kern/uipc_usrreq.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/kern/uipc_usrreq.c	2006/11/30 20:59:33
@@ -568,22 +568,23 @@
 		 * Adjust backpressure on sender and wakeup any waiting to
 		 * write.
 		 */
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		mbcnt = so->so_rcv.sb_mbcnt;
 		sbcc = so->so_rcv.sb_cc;
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		UNP_LOCK();
 		if (unp->unp_conn == NULL) {
 			UNP_UNLOCK();
 			break;
 		}
 		so2 = unp->unp_conn->unp_socket;
-		SOCKBUF_LOCK(&so2->so_snd);
+		SOCK_LOCK(so2);
 		so2->so_snd.sb_mbmax += unp->unp_mbcnt - mbcnt;
 		newhiwat = so2->so_snd.sb_hiwat + unp->unp_cc - sbcc;
 		(void)chgsbsize(so2->so_cred->cr_uidinfo, &so2->so_snd.sb_hiwat,
 		    newhiwat, RLIM_INFINITY);
 		sowwakeup_locked(so2);
+		SOCK_UNLOCK(so2);
 		unp->unp_mbcnt = mbcnt;
 		unp->unp_cc = sbcc;
 		UNP_UNLOCK();
@@ -651,15 +652,14 @@
 			from = &sun_noname;
 		if (unp2->unp_flags & UNP_WANTCRED)
 			control = unp_addsockcred(td, control);
-		SOCKBUF_LOCK(&so2->so_rcv);
+		SOCK_LOCK(so2);
 		if (sbappendaddr_locked(&so2->so_rcv, from, m, control)) {
 			sorwakeup_locked(so2);
 			m = NULL;
 			control = NULL;
-		} else {
-			SOCKBUF_UNLOCK(&so2->so_rcv);
+		} else
 			error = ENOBUFS;
-		}
+		SOCK_UNLOCK(so2);
 		if (nam != NULL)
 			unp_disconnect(unp);
 		break;
@@ -671,6 +671,8 @@
 		 *
 		 * Note: A better implementation would complain if not equal
 		 * to the peer's address.
+		 *
+		 * XXXRW: Lockless read.
 		 */
 		if ((so->so_state & SS_ISCONNECTED) == 0) {
 			if (nam != NULL) {
@@ -701,7 +703,6 @@
 			break;
 		}
 		so2 = unp2->unp_socket;
-		SOCKBUF_LOCK(&so2->so_rcv);
 		if (unp2->unp_flags & UNP_WANTCRED) {
 			/*
 			 * Credentials are passed only once on
@@ -714,6 +715,7 @@
 		 * Send to paired receive port, and then reduce send buffer
 		 * hiwater marks to maintain backpressure.  Wake up readers.
 		 */
+		SOCK_LOCK(so2);
 		if (control != NULL) {
 			if (sbappendcontrol_locked(&so2->so_rcv, m, control))
 				control = NULL;
@@ -724,13 +726,13 @@
 		unp2->unp_mbcnt = so2->so_rcv.sb_mbcnt;
 		sbcc = so2->so_rcv.sb_cc;
 		sorwakeup_locked(so2);
-
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_UNLOCK(so2);
+		SOCK_LOCK(so);
 		newhiwat = so->so_snd.sb_hiwat - (sbcc - unp2->unp_cc);
 		(void)chgsbsize(so->so_cred->cr_uidinfo, &so->so_snd.sb_hiwat,
 		    newhiwat, RLIM_INFINITY);
 		so->so_snd.sb_mbmax -= mbcnt;
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 
 		unp2->unp_cc = sbcc;
 		m = NULL;
@@ -1700,9 +1702,9 @@
 			 * message buffers. Follow those links and mark them
 			 * as accessible too.
 			 */
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 			unp_scan(so->so_rcv.sb_mb, unp_mark);
-			SOCKBUF_UNLOCK(&so->so_rcv);
+			SOCK_UNLOCK(so);
 		}
 	} while (unp_defer);
 	sx_sunlock(&filelist_lock);
--- //depot/user/rwatson/sockref/src/sys/kern/vfs_aio.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/kern/vfs_aio.c	2006/11/30 15:30:26
@@ -1297,7 +1297,7 @@
 	else
 		opcode = LIO_READ;
 
-	SOCKBUF_LOCK(sb);
+	SOCK_LOCK(so);
 	sb->sb_flags &= ~SB_AIO;
 	mtx_lock(&aio_job_mtx);
 	TAILQ_FOREACH_SAFE(cb, &so->so_aiojobq, list, cbn) {
@@ -1315,7 +1315,7 @@
 		}
 	}
 	mtx_unlock(&aio_job_mtx);
-	SOCKBUF_UNLOCK(sb);
+	SOCK_UNLOCK(so);
 }
 
 /*
@@ -1494,7 +1494,7 @@
 		 */
 		so = fp->f_data;
 		sb = (opcode == LIO_READ) ? &so->so_rcv : &so->so_snd;
-		SOCKBUF_LOCK(sb);
+		SOCK_LOCK(so);
 		if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
 		    LIO_WRITE) && (!sowriteable(so)))) {
 			sb->sb_flags |= SB_AIO;
@@ -1511,12 +1511,12 @@
 			if (lj)
 				lj->lioj_count++;
 			AIO_UNLOCK(ki);
-			SOCKBUF_UNLOCK(sb);
+			SOCK_UNLOCK(so);
 			atomic_add_int(&num_queue_count, 1);
 			error = 0;
 			goto done;
 		}
-		SOCKBUF_UNLOCK(sb);
+		SOCK_UNLOCK(so);
 	}
 
 	if ((error = aio_qphysio(p, aiocbe)) == 0)
--- //depot/user/rwatson/sockref/src/sys/netatalk/ddp_input.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netatalk/ddp_input.c	2006/11/30 15:30:26
@@ -422,10 +422,10 @@
     /* 
      * If we found one, deliver the packet to the socket
      */
-    SOCKBUF_LOCK(&ddp->ddp_socket->so_rcv);
+    SOCK_LOCK(ddp->ddp_socket);
     if (sbappendaddr_locked(&ddp->ddp_socket->so_rcv, (struct sockaddr *)&from,
 	    m, NULL) == 0) {
-    	SOCKBUF_UNLOCK(&ddp->ddp_socket->so_rcv);
+	SOCK_UNLOCK(ddp->ddp_socket);
 	/* 
 	 * If the socket is full (or similar error) dump the packet.
 	 */
@@ -436,6 +436,7 @@
      * And wake up whatever might be waiting for it
      */
     sorwakeup_locked(ddp->ddp_socket);
+    SOCK_UNLOCK(ddp->ddp_socket);
     m = NULL;
 out:
     DDP_LIST_SUNLOCK();
--- //depot/user/rwatson/sockref/src/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c	2006/11/30 15:30:26
@@ -1008,12 +1008,10 @@
 			/* Close L2CAP socket */
 			s->l2so->so_upcallarg = NULL;
 			s->l2so->so_upcall = NULL;
-			SOCKBUF_LOCK(&s->l2so->so_rcv);
+			SOCK_LOCK(s->l2so);
 			s->l2so->so_rcv.sb_flags &= ~SB_UPCALL;
-			SOCKBUF_UNLOCK(&s->l2so->so_rcv);
-			SOCKBUF_LOCK(&s->l2so->so_snd);
 			s->l2so->so_snd.sb_flags &= ~SB_UPCALL;
-			SOCKBUF_UNLOCK(&s->l2so->so_snd);
+			SOCK_UNLOCK(s->l2so);
 			soclose(s->l2so);
 
 			mtx_unlock(&s->session_mtx);
@@ -1247,13 +1245,11 @@
 	/* Prepare L2CAP socket */
 	l2so->so_upcallarg = NULL;
 	l2so->so_upcall = ng_btsocket_rfcomm_upcall;
-	SOCKBUF_LOCK(&l2so->so_rcv);
+	SOCK_LOCK(l2so);
 	l2so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&l2so->so_rcv);
-	SOCKBUF_LOCK(&l2so->so_snd);
 	l2so->so_snd.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&l2so->so_snd);
 	l2so->so_state |= SS_NBIO;
+	SOCK_UNLOCK(l2so);
 	s->l2so = l2so;
 
 	mtx_lock(&s->session_mtx);
@@ -1329,15 +1325,13 @@
 	mtx_unlock(&s->session_mtx);
 
 	/* Return L2CAP socket back to its original state */
+	SOCK_LOCK(l2so);
 	l2so->so_upcallarg = NULL;
 	l2so->so_upcall = NULL;
-	SOCKBUF_LOCK(&l2so->so_rcv);
 	l2so->so_rcv.sb_flags &= ~SB_UPCALL;
-	SOCKBUF_UNLOCK(&l2so->so_rcv);
-	SOCKBUF_LOCK(&l2so->so_snd);
 	l2so->so_snd.sb_flags &= ~SB_UPCALL;
-	SOCKBUF_UNLOCK(&l2so->so_snd);
 	l2so->so_state &= ~SS_NBIO;
+	SOCK_UNLOCK(l2so);
 
 	mtx_destroy(&s->session_mtx);
 	bzero(s, sizeof(*s));
--- //depot/user/rwatson/sockref/src/sys/netgraph/ng_ksocket.c	2006/07/24 16:29:14
+++ //depot/user/rwatson/resock/src/sys/netgraph/ng_ksocket.c	2006/07/24 17:41:35
@@ -615,15 +615,11 @@
 	struct socket *const so = priv->so;
 
 	/* Add our hook for incoming data and other events */
+	SOCK_LOCK(priv->so);
 	priv->so->so_upcallarg = (caddr_t)node;
 	priv->so->so_upcall = ng_ksocket_incoming;
-	SOCKBUF_LOCK(&priv->so->so_rcv);
 	priv->so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&priv->so->so_rcv);
-	SOCKBUF_LOCK(&priv->so->so_snd);
 	priv->so->so_snd.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&priv->so->so_snd);
-	SOCK_LOCK(priv->so);
 	priv->so->so_state |= SS_NBIO;
 	SOCK_UNLOCK(priv->so);
 	/*
@@ -936,13 +932,11 @@
 
 	/* Close our socket (if any) */
 	if (priv->so != NULL) {
+		SOCK_LOCK(priv->so);
 		priv->so->so_upcall = NULL;
-		SOCKBUF_LOCK(&priv->so->so_rcv);
 		priv->so->so_rcv.sb_flags &= ~SB_UPCALL;
-		SOCKBUF_UNLOCK(&priv->so->so_rcv);
-		SOCKBUF_LOCK(&priv->so->so_snd);
 		priv->so->so_snd.sb_flags &= ~SB_UPCALL;
-		SOCKBUF_UNLOCK(&priv->so->so_snd);
+		SOCK_UNLOCK(priv->so);
 		soclose(priv->so);
 		priv->so = NULL;
 	}
@@ -1261,14 +1255,12 @@
 	 */
 	LIST_INSERT_HEAD(&priv->embryos, priv2, siblings);
 
+	SOCK_LOCK(so);
 	so->so_upcallarg = (caddr_t)node;
 	so->so_upcall = ng_ksocket_incoming;
-	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 
 	/* Fill in the response data and send it or return it to the caller */
 	resp_data = (struct ng_ksocket_accept *)resp->data;
--- //depot/user/rwatson/sockref/src/sys/netinet/ip_divert.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/ip_divert.c	2006/11/30 15:30:26
@@ -266,14 +266,14 @@
 		/* XXX why does only one socket match? */
 		if (inp->inp_lport == nport) {
 			sa = inp->inp_socket;
-			SOCKBUF_LOCK(&sa->so_rcv);
+			SOCK_LOCK(sa);
 			if (sbappendaddr_locked(&sa->so_rcv,
 			    (struct sockaddr *)&divsrc, m,
-			    (struct mbuf *)0) == 0) {
-				SOCKBUF_UNLOCK(&sa->so_rcv);
+			    (struct mbuf *)0) == 0)
 				sa = NULL;	/* force mbuf reclaim below */
-			} else
+			else
 				sorwakeup_locked(sa);
+			SOCK_UNLOCK(sa);
 			INP_UNLOCK(inp);
 			break;
 		}
--- //depot/user/rwatson/sockref/src/sys/netinet/ip_mroute.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/ip_mroute.c	2006/11/30 15:30:26
@@ -1437,13 +1437,14 @@
 socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
 {
     if (s) {
-	SOCKBUF_LOCK(&s->so_rcv);
+	SOCK_LOCK(s);
 	if (sbappendaddr_locked(&s->so_rcv, (struct sockaddr *)src, mm,
 	    NULL) != 0) {
 	    sorwakeup_locked(s);
+	    SOCK_UNLOCK(s);
 	    return 0;
 	}
-	SOCKBUF_UNLOCK(&s->so_rcv);
+	SOCK_UNLOCK(s);
     }
     m_freem(mm);
     return -1;
--- //depot/user/rwatson/sockref/src/sys/netinet/raw_ip.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/raw_ip.c	2006/11/30 15:30:26
@@ -187,16 +187,16 @@
 		if ((last->inp_flags & INP_CONTROLOPTS) ||
 		    (so->so_options & (SO_TIMESTAMP | SO_BINTIME)))
 			ip_savecontrol(last, &opts, ip, n);
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		if (sbappendaddr_locked(&so->so_rcv,
 		    (struct sockaddr *)&ripsrc, n, opts) == 0) {
 			/* should notify about lost packet */
 			m_freem(n);
 			if (opts)
 				m_freem(opts);
-			SOCKBUF_UNLOCK(&so->so_rcv);
 		} else
 			sorwakeup_locked(so);
+		SOCK_UNLOCK(so);
 	} else
 		m_freem(n);
 	return policyfail;
--- //depot/user/rwatson/sockref/src/sys/netinet/sctp_constants.h	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/sctp_constants.h	2007/04/29 12:41:48
@@ -946,8 +946,8 @@
 
 #define sctp_sowwakeup_locked(inp, so) \
 do { \
+	SOCK_LOCK_ASSERT(so); \
 	if (inp->sctp_flags & SCTP_PCB_FLAGS_DONT_WAKE) { \
-                SOCKBUF_UNLOCK(&((so)->so_snd)); \
 		inp->sctp_flags |= SCTP_PCB_FLAGS_WAKEOUTPUT; \
 	} else { \
 		sowwakeup_locked(so); \
@@ -966,9 +966,9 @@
 /* FIXME */
 #define sctp_sorwakeup_locked(inp, so) \
 do { \
+	SOCK_LOCK_ASSERT(so); \
 	if (inp->sctp_flags & SCTP_PCB_FLAGS_DONT_WAKE) { \
 		inp->sctp_flags |= SCTP_PCB_FLAGS_WAKEINPUT; \
-                SOCKBUF_UNLOCK(&((so)->so_rcv)); \
 	} else { \
 		sorwakeup_locked(so); \
 	} \
--- //depot/user/rwatson/sockref/src/sys/netinet/sctp_indata.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/sctp_indata.c	2007/04/29 12:41:48
@@ -4181,11 +4181,12 @@
 		tp1 = tp2;
 	}
 	if (stcb->sctp_socket) {
-		SOCKBUF_LOCK(&stcb->sctp_socket->so_snd);
+		SOCK_LOCK(stcb->sctp_socket);
 #ifdef SCTP_WAKE_LOGGING
 		sctp_wakeup_log(stcb, cumack, 1, SCTP_WAKESND_FROM_SACK);
 #endif
 		sctp_sowwakeup_locked(stcb->sctp_ep, stcb->sctp_socket);
+		SOCK_UNLOCK(stcb->sctp_socket);
 #ifdef SCTP_WAKE_LOGGING
 	} else {
 		sctp_wakeup_log(stcb, cumack, 1, SCTP_NOWAKE_FROM_SACK);
@@ -4850,11 +4851,12 @@
 
 done_with_it:
 	if ((wake_him) && (stcb->sctp_socket)) {
-		SOCKBUF_LOCK(&stcb->sctp_socket->so_snd);
+		SOCK_LOCK(stcb->sctp_socket);
 #ifdef SCTP_WAKE_LOGGING
 		sctp_wakeup_log(stcb, cum_ack, wake_him, SCTP_WAKESND_FROM_SACK);
 #endif
 		sctp_sowwakeup_locked(stcb->sctp_ep, stcb->sctp_socket);
+		SOCK_UNLOCK(stcb->sctp_socket);
 #ifdef SCTP_WAKE_LOGGING
 	} else {
 		sctp_wakeup_log(stcb, cum_ack, wake_him, SCTP_NOWAKE_FROM_SACK);
--- //depot/user/rwatson/sockref/src/sys/netinet/sctp_output.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/sctp_output.c	2007/04/29 12:41:48
@@ -9840,7 +9840,7 @@
 	len = 0;
 	if (max_len < sctp_add_more_threshold) {
 		/* No room right no ! */
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_LOCK(so);
 		while (so->so_snd.sb_hiwat < (stcb->asoc.total_output_queue_size + sctp_add_more_threshold)) {
 #ifdef SCTP_BLK_LOGGING
 			sctp_log_block(SCTP_BLOCK_LOG_INTO_BLKA,
@@ -9858,7 +9858,7 @@
 						error = be.error;
 					}
 				}
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				goto out_unlocked;
 			}
 #ifdef SCTP_BLK_LOGGING
@@ -9874,7 +9874,7 @@
 		} else {
 			max_len = 0;
 		}
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 	}
 	if (stcb->asoc.state & SCTP_STATE_ABOUT_TO_BE_FREED) {
 		goto out_unlocked;
@@ -10146,7 +10146,7 @@
 					hold_tcblock = 0;
 				}
 			}
-			SOCKBUF_LOCK(&so->so_snd);
+			SOCK_LOCK(so);
 			/*
 			 * This is a bit strange, but I think it will work.
 			 * The total_output_queue_size is locked and
@@ -10180,7 +10180,7 @@
 							error = be.error;
 						}
 					}
-					SOCKBUF_UNLOCK(&so->so_snd);
+					SOCK_UNLOCK(so);
 					goto out_unlocked;
 				}
 #ifdef SCTP_BLK_LOGGING
@@ -10188,7 +10188,7 @@
 				    so, asoc, stcb->asoc.total_output_queue_size);
 #endif
 			}
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			if (stcb->asoc.state & SCTP_STATE_ABOUT_TO_BE_FREED) {
 				goto out_unlocked;
 			}
--- //depot/user/rwatson/sockref/src/sys/netinet/sctp_usrreq.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/sctp_usrreq.c	2007/04/29 12:41:48
@@ -4867,22 +4867,20 @@
 		inp->sctp_flags &= ~SCTP_PCB_FLAGS_DONT_WAKE;
 		if (inp->sctp_flags & SCTP_PCB_FLAGS_WAKEOUTPUT) {
 			inp->sctp_flags &= ~SCTP_PCB_FLAGS_WAKEOUTPUT;
-			SOCKBUF_LOCK(&inp->sctp_socket->so_snd);
+			SOCK_LOCK(inp->sctp_socket);
 			if (sowriteable(inp->sctp_socket)) {
 				sowwakeup_locked(inp->sctp_socket);
-			} else {
-				SOCKBUF_UNLOCK(&inp->sctp_socket->so_snd);
 			}
+			SOCK_UNLOCK(inp->sctp_socket);
 		}
 		if (inp->sctp_flags & SCTP_PCB_FLAGS_WAKEINPUT) {
 			inp->sctp_flags &= ~SCTP_PCB_FLAGS_WAKEINPUT;
-			SOCKBUF_LOCK(&inp->sctp_socket->so_rcv);
+			SOCK_LOCK(inp->sctp_socket);
 			if (soreadable(inp->sctp_socket)) {
 				sctp_defered_wakeup_cnt++;
 				sorwakeup_locked(inp->sctp_socket);
-			} else {
-				SOCKBUF_UNLOCK(&inp->sctp_socket->so_rcv);
 			}
+			SOCK_UNLOCK(inp->sctp_socket);
 		}
 	}
 	splx(s);
--- //depot/user/rwatson/sockref/src/sys/netinet/sctputil.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/sctputil.c	2007/04/29 12:41:48
@@ -402,9 +402,9 @@
 	}
 	sctp_clog[sctp_cwnd_log_at].x.lock.info_lock = mtx_owned(&sctppcbinfo.ipi_ep_mtx);
 	if (inp->sctp_socket) {
-		sctp_clog[sctp_cwnd_log_at].x.lock.sock_lock = mtx_owned(&(inp->sctp_socket->so_rcv.sb_mtx));
-		sctp_clog[sctp_cwnd_log_at].x.lock.sockrcvbuf_lock = mtx_owned(&(inp->sctp_socket->so_rcv.sb_mtx));
-		sctp_clog[sctp_cwnd_log_at].x.lock.socksndbuf_lock = mtx_owned(&(inp->sctp_socket->so_snd.sb_mtx));
+		sctp_clog[sctp_cwnd_log_at].x.lock.sock_lock = mtx_owned(&(inp->sctp_socket->so_mtx));
+		sctp_clog[sctp_cwnd_log_at].x.lock.sockrcvbuf_lock = mtx_owned(&(inp->sctp_socket->so_mtx));
+		sctp_clog[sctp_cwnd_log_at].x.lock.socksndbuf_lock = mtx_owned(&(inp->sctp_socket->so_mtx));
 	} else {
 		sctp_clog[sctp_cwnd_log_at].x.lock.sock_lock = SCTP_LOCK_UNKNOWN;
 		sctp_clog[sctp_cwnd_log_at].x.lock.sockrcvbuf_lock = SCTP_LOCK_UNKNOWN;
@@ -3695,11 +3695,11 @@
 	new_so = new_inp->sctp_socket;
 	TAILQ_INIT(&tmp_queue);
 
-	SOCKBUF_LOCK(&(old_so->so_rcv));
+	SOCK_LOCK(old_so);
 
 	error = sblock(&old_so->so_rcv, 0);
 
-	SOCKBUF_UNLOCK(&(old_so->so_rcv));
+	SOCK_UNLOCK(old_so);
 	if (error) {
 		/*
 		 * Gak, can't get sblock, we have a problem. data will be
@@ -3739,10 +3739,10 @@
 	SCTP_INP_READ_UNLOCK(old_inp);
 
 	/* Remove the sb-lock on the old socket */
-	SOCKBUF_LOCK(&(old_so->so_rcv));
+	SOCK_LOCK(old_so);
 
 	sbunlock(&old_so->so_rcv);
-	SOCKBUF_UNLOCK(&(old_so->so_rcv));
+	SOCK_UNLOCK(old_so);
 
 	/* Now we move them over to the new socket buffer */
 	control = TAILQ_FIRST(&tmp_queue);
@@ -4327,7 +4327,7 @@
 	sctp_misc_ints(SCTP_SORECV_ENTER,
 	    rwnd_req, in_eeor_mode, so->so_rcv.sb_cc, uio->uio_resid);
 #endif
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	hold_sblock = 1;
 #ifdef SCTP_RECV_RWND_LOGGING
 	sctp_misc_ints(SCTP_SORECV_ENTERPL,
@@ -4341,14 +4341,14 @@
 	}
 restart:
 	if (hold_sblock == 0) {
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		hold_sblock = 1;
 	}
 	sbunlock(&so->so_rcv);
 
 restart_nosblocks:
 	if (hold_sblock == 0) {
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		hold_sblock = 1;
 	}
 	if ((inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE) ||
@@ -4520,7 +4520,7 @@
 	 * Note that stcb COULD be NULL.
 	 */
 	if (hold_sblock) {
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		hold_sblock = 0;
 	}
 	stcb = control->stcb;
@@ -4956,7 +4956,7 @@
 			hold_rlock = 0;
 		}
 		if (hold_sblock == 0) {
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 			hold_sblock = 1;
 		}
 #ifdef SCTP_RECV_DETAIL_RWND_LOGGING
@@ -4981,7 +4981,7 @@
 			control->held_length = 0;
 		}
 		if (hold_sblock) {
-			SOCKBUF_UNLOCK(&so->so_rcv);
+			SOCK_UNLOCK(so);
 			hold_sblock = 0;
 		}
 		if (control->length == 0) {
@@ -5080,7 +5080,7 @@
 				hold_rlock = 0;
 			}
 			if (hold_sblock == 0) {
-				SOCKBUF_LOCK(&so->so_rcv);
+				SOCK_LOCK(so);
 				hold_sblock = 1;
 			}
 			if (so->so_rcv.sb_cc <= control->held_length) {
@@ -5090,7 +5090,7 @@
 				}
 			}
 			if (hold_sblock) {
-				SOCKBUF_UNLOCK(&so->so_rcv);
+				SOCK_UNLOCK(so);
 				hold_sblock = 0;
 			}
 			if (control->length == 0) {
@@ -5163,7 +5163,7 @@
 						hold_rlock = 0;
 					}
 					if (hold_sblock) {
-						SOCKBUF_UNLOCK(&so->so_rcv);
+						SOCK_UNLOCK(so);
 						hold_sblock = 0;
 					}
 					splx(s);
@@ -5175,7 +5175,7 @@
 					sctp_log_lock(inp, stcb, SCTP_LOG_LOCK_SOCKBUF_R);
 #endif
 					if (hold_sblock == 0) {
-						SOCKBUF_LOCK(&so->so_rcv);
+						SOCK_LOCK(so);
 						hold_sblock = 1;
 					}
 					if (inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE)
@@ -5221,14 +5221,14 @@
 		hold_rlock = 0;
 	}
 	if (hold_sblock == 0) {
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		hold_sblock = 1;
 	}
 	sbunlock(&so->so_rcv);
 
 release_unlocked:
 	if (hold_sblock) {
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		hold_sblock = 0;
 	}
 	if ((stcb) && (in_flags & MSG_PEEK) == 0) {
@@ -5245,7 +5245,7 @@
 		hold_rlock = 0;
 	}
 	if (hold_sblock) {
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		hold_sblock = 0;
 	}
 	if (freecnt_applied) {
--- //depot/user/rwatson/sockref/src/sys/netinet/tcp_input.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet/tcp_input.c	2007/04/29 12:41:48
@@ -372,7 +372,7 @@
 	q = LIST_FIRST(&tp->t_segq);
 	if (!q || q->tqe_th->th_seq != tp->rcv_nxt)
 		return (0);
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	do {
 		tp->rcv_nxt += q->tqe_len;
 		flags = q->tqe_th->th_flags & TH_FIN;
@@ -389,6 +389,7 @@
 	} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
 	ND6_HINT(tp);
 	sorwakeup_locked(so);
+	SOCK_UNLOCK(so);
 	return (flags);
 }
 
@@ -1329,7 +1330,7 @@
 #endif
 			 * Add data to socket buffer.
 			 */
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 				m_freem(m);
 			} else {
@@ -1337,6 +1338,7 @@
 				sbappendstream_locked(&so->so_rcv, m);
 			}
 			sorwakeup_locked(so);
+			SOCK_UNLOCK(so);
 			if (DELAY_ACK(tp)) {
 				tp->t_flags |= TF_DELACK;
 			} else {
@@ -2173,7 +2175,7 @@
 				incr = incr * incr / cw;
 			tp->snd_cwnd = min(cw+incr, TCP_MAXWIN<<tp->snd_scale);
 		}
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_LOCK(so);
 		if (acked > so->so_snd.sb_cc) {
 			tp->snd_wnd -= so->so_snd.sb_cc;
 			sbdrop_locked(&so->so_snd, (int)so->so_snd.sb_cc);
@@ -2184,6 +2186,7 @@
 			ourfinisacked = 0;
 		}
 		sowwakeup_locked(so);
+		SOCK_UNLOCK(so);
 		/* detect una wraparound */
 		if ((tcp_do_newreno || tp->sack_enable) &&
 		    !IN_FASTRECOVERY(tp) &&
@@ -2311,11 +2314,11 @@
 		 * soreceive.  It's hard to imagine someone
 		 * actually wanting to send this much urgent data.
 		 */
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		if (th->th_urp + so->so_rcv.sb_cc > sb_max) {
 			th->th_urp = 0;			/* XXX */
 			thflags &= ~TH_URG;		/* XXX */
-			SOCKBUF_UNLOCK(&so->so_rcv);	/* XXX */
+			SOCK_UNLOCK(so);		/* XXX */
 			goto dodata;			/* XXX */
 		}
 		/*
@@ -2341,7 +2344,7 @@
 			sohasoutofband(so);
 			tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
 		}
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		/*
 		 * Remove out of band data so doesn't get presented to user.
 		 * This can happen independent of advancing the URG pointer,
@@ -2403,12 +2406,13 @@
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
-			SOCKBUF_LOCK(&so->so_rcv);
+			SOCK_LOCK(so);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
 			else
 				sbappendstream_locked(&so->so_rcv, m);
 			sorwakeup_locked(so);
+			SOCK_UNLOCK(so);
 		} else {
 			thflags = tcp_reass(tp, th, &tlen, m);
 			tp->t_flags |= TF_ACKNOW;
@@ -2998,7 +3002,7 @@
 	 * Make the socket buffers an integral number of mss units;
 	 * if the mss is larger than the socket buffer, decrease the mss.
 	 */
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	if ((so->so_snd.sb_hiwat == tcp_sendspace) && metrics.rmx_sendpipe)
 		bufsize = metrics.rmx_sendpipe;
 	else
@@ -3012,10 +3016,8 @@
 		if (bufsize > so->so_snd.sb_hiwat)
 			(void)sbreserve_locked(&so->so_snd, bufsize, so, NULL);
 	}
-	SOCKBUF_UNLOCK(&so->so_snd);
 	tp->t_maxseg = mss;
 
-	SOCKBUF_LOCK(&so->so_rcv);
 	if ((so->so_rcv.sb_hiwat == tcp_recvspace) && metrics.rmx_recvpipe)
 		bufsize = metrics.rmx_recvpipe;
 	else
@@ -3027,7 +3029,7 @@
 		if (bufsize > so->so_rcv.sb_hiwat)
 			(void)sbreserve_locked(&so->so_rcv, bufsize, so, NULL);
 	}
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 	/*
 	 * While we're here, check the others too
 	 */
--- //depot/user/rwatson/sockref/src/sys/netinet/tcp_output.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/tcp_output.c	2006/11/30 15:30:26
@@ -257,7 +257,7 @@
 	if (tp->t_flags & TF_NEEDSYN)
 		flags |= TH_SYN;
 
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	/*
 	 * If in persist timeout with window of 0, send 1 byte.
 	 * Otherwise, if window is small but nonzero
@@ -544,11 +544,11 @@
 	 * No reason to send a segment, just return.
 	 */
 just_return:
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 	return (0);
 
 send:
-	SOCKBUF_LOCK_ASSERT(&so->so_snd);
+	SOCK_LOCK_ASSERT(so);
 	/*
 	 * Before ESTABLISHED, force sending of initial options
 	 * unless TCP set not to do any options.
@@ -779,7 +779,7 @@
 #ifdef notyet
 		if ((m = m_copypack(so->so_snd.sb_mb, off,
 		    (int)len, max_linkhdr + hdrlen)) == 0) {
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			error = ENOBUFS;
 			goto out;
 		}
@@ -791,7 +791,7 @@
 #else
 		MGETHDR(m, M_DONTWAIT, MT_DATA);
 		if (m == NULL) {
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			error = ENOBUFS;
 			goto out;
 		}
@@ -799,7 +799,7 @@
 		if (MHLEN < hdrlen + max_linkhdr) {
 			MCLGET(m, M_DONTWAIT);
 			if ((m->m_flags & M_EXT) == 0) {
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				m_freem(m);
 				error = ENOBUFS;
 				goto out;
@@ -815,7 +815,7 @@
 		} else {
 			m->m_next = m_copy(so->so_snd.sb_mb, off, (int) len);
 			if (m->m_next == 0) {
-				SOCKBUF_UNLOCK(&so->so_snd);
+				SOCK_UNLOCK(so);
 				(void) m_free(m);
 				error = ENOBUFS;
 				goto out;
@@ -830,9 +830,9 @@
 		 */
 		if (off + len == so->so_snd.sb_cc)
 			flags |= TH_PUSH;
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 	} else {
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 		if (tp->t_flags & TF_ACKNOW)
 			tcpstat.tcps_sndacks++;
 		else if (flags & (TH_SYN|TH_FIN|TH_RST))
@@ -856,7 +856,7 @@
 		m->m_data += max_linkhdr;
 		m->m_len = hdrlen;
 	}
-	SOCKBUF_UNLOCK_ASSERT(&so->so_snd);
+	SOCK_UNLOCK_ASSERT(so);
 	m->m_pkthdr.rcvif = (struct ifnet *)0;
 #ifdef MAC
 	mac_create_mbuf_from_inpcb(tp->t_inpcb, m);
@@ -1171,7 +1171,7 @@
 		}
 
 out:
-		SOCKBUF_UNLOCK_ASSERT(&so->so_snd);	/* Check gotos. */
+		SOCK_UNLOCK_ASSERT(so);	/* Check gotos. */
 		if (error == ENOBUFS) {
 	                if (!callout_active(tp->tt_rexmt) &&
 			    !callout_active(tp->tt_persist))
--- //depot/user/rwatson/sockref/src/sys/netinet/tcp_usrreq.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/tcp_usrreq.c	2006/11/30 15:30:26
@@ -254,6 +254,15 @@
 	    ("tcp_usr_detach: inp_socket == NULL"));
 	TCPDEBUG1();
 
+	/*
+	 * First, if we still have full TCP state, and we're not dropped,
+	 * initiate a disconnect.
+	 */
+	if (!(inp->inp_vflag & INP_TIMEWAIT) &&
+	    !(inp->inp_vflag & INP_DROPPED)) {
+		tp = intotcpcb(inp);
+		tcp_disconnect(tp);
+	}
 	tcp_detach(so, inp);
 	tp = NULL;
 	TCPDEBUG2(PRU_DETACH);
@@ -868,9 +877,9 @@
 		/*
 		 * XXXRW: PRUS_EOF not implemented with PRUS_OOB?
 		 */
-		SOCKBUF_LOCK(&so->so_snd);
+		SOCK_LOCK(so);
 		if (sbspace(&so->so_snd) < -512) {
-			SOCKBUF_UNLOCK(&so->so_snd);
+			SOCK_UNLOCK(so);
 			m_freem(m);
 			error = ENOBUFS;
 			goto out;
@@ -884,7 +893,7 @@
 		 * Otherwise, snd_up should be one lower.
 		 */
 		sbappendstream_locked(&so->so_snd, m);
-		SOCKBUF_UNLOCK(&so->so_snd);
+		SOCK_UNLOCK(so);
 		if (nam && tp->t_state < TCPS_SYN_SENT) {
 			/*
 			 * Do implied connect if not yet connected,
@@ -925,6 +934,8 @@
 
 /*
  * Abort the TCP.  Drop the connection abruptly.
+ *
+ * First, drop the connection.  Then collect state if possible.
  */
 static void
 tcp_usr_abort(struct socket *so)
--- //depot/user/rwatson/sockref/src/sys/netinet/udp_usrreq.c	2006/11/30 09:41:02
+++ //depot/user/rwatson/resock/src/sys/netinet/udp_usrreq.c	2006/11/30 15:30:26
@@ -508,15 +508,15 @@
 	m_adj(n, off);
 
 	so = inp->inp_socket;
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	if (sbappendaddr_locked(&so->so_rcv, append_sa, n, opts) == 0) {
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
 		udpstat.udps_fullsock++;
-		SOCKBUF_UNLOCK(&so->so_rcv);
 	} else
 		sorwakeup_locked(so);
+	SOCK_UNLOCK(so);
 }
 
 /*
--- //depot/user/rwatson/sockref/src/sys/netinet6/udp6_usrreq.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/netinet6/udp6_usrreq.c	2007/04/29 12:41:48
@@ -149,16 +149,16 @@
 	m_adj(n, off + sizeof(struct udphdr));
 
 	so = in6p->inp_socket;
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	if (sbappendaddr_locked(&so->so_rcv, (struct sockaddr *)fromsa, n,
 	    opts) == 0) {
-		SOCKBUF_UNLOCK(&so->so_rcv);
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
 		udpstat.udps_fullsock++;
 	} else
 		sorwakeup_locked(so);
+	SOCK_UNLOCK(so);
 }
 
 int
--- //depot/user/rwatson/sockref/src/sys/netipx/spx_usrreq.c	2006/07/22 15:57:44
+++ //depot/user/rwatson/resock/src/sys/netipx/spx_usrreq.c	2006/07/22 17:07:41
@@ -526,7 +526,7 @@
 	/*
 	 * Trim Acked data from output queue.
 	 */
-	SOCKBUF_LOCK(&so->so_snd);
+	SOCK_LOCK(so);
 	while ((m = so->so_snd.sb_mb) != NULL) {
 		if (SSEQ_LT((mtod(m, struct spx *))->si_seq, si->si_ack))
 			sbdroprecord_locked(&so->so_snd);
@@ -534,6 +534,7 @@
 			break;
 	}
 	sowwakeup_locked(so);
+	SOCK_UNLOCK(so);
 	cb->s_rack = si->si_ack;
 update_window:
 	if (SSEQ_LT(cb->s_snxt, cb->s_rack))
@@ -634,7 +635,7 @@
 	}
 present:
 #define SPINC sizeof(struct spxhdr)
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 
 	/*
 	 * Loop through all packets queued up to update acknowledge number,
@@ -708,8 +709,7 @@
 	}
 	if (wakeup)
 		sorwakeup_locked(so);
-	else
-		SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 	return (0);
 }
 
@@ -1554,6 +1554,9 @@
 	IPX_LOCK(ipxp);
 	spx_pcbdetach(ipxp);
 	ipx_pcbfree(ipxp);
+	spx_pcbdetach(ipxp);
+	ipx_pcbdetach(ipxp);
+	ipx_pcbfree(ipxp);
 	IPX_LIST_UNLOCK();
 }
 
@@ -1671,16 +1674,16 @@
 		error = EINVAL;
 		goto out;
 	}
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	if ((cb->s_oobflags & SF_IOOB) || so->so_oobmark ||
 	    (so->so_rcv.sb_state & SBS_RCVATMARK)) {
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		m->m_len = 1;
 		*mtod(m, caddr_t) = cb->s_iobc;
 		error = 0;
 		goto out;
 	}
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 	error = EINVAL;
 out:
 	IPX_UNLOCK(ipxp);
--- //depot/user/rwatson/sockref/src/sys/netsmb/smb_trantcp.c	2006/08/06 10:52:58
+++ //depot/user/rwatson/resock/src/sys/netsmb/smb_trantcp.c	2006/11/30 15:30:26
@@ -223,51 +223,52 @@
 nb_connect_in(struct nbpcb *nbp, struct sockaddr_in *to, struct thread *td)
 {
 	struct socket *so;
-	int error, s;
+	int error;
 
 	error = socreate(AF_INET, &so, SOCK_STREAM, IPPROTO_TCP,
 	    td->td_ucred, td);
 	if (error)
 		return error;
 	nbp->nbp_tso = so;
+	SOCK_LOCK(so);
 	so->so_upcallarg = (caddr_t)nbp;
 	so->so_upcall = nb_upcall;
-	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&so->so_rcv);
 	so->so_rcv.sb_timeo = (5 * hz);
 	so->so_snd.sb_timeo = (5 * hz);
+	SOCK_UNLOCK(so);
 	error = soreserve(so, nbp->nbp_sndbuf, nbp->nbp_rcvbuf);
 	if (error)
 		goto bad;
 	nb_setsockopt_int(so, SOL_SOCKET, SO_KEEPALIVE, 1);
 	nb_setsockopt_int(so, IPPROTO_TCP, TCP_NODELAY, 1);
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_rcv.sb_flags &= ~SB_NOINTR;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_flags &= ~SB_NOINTR;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 	error = soconnect(so, (struct sockaddr*)to, td);
 	if (error)
 		goto bad;
-	s = splnet();
+	/*
+	 * XXXRW: Move to a properly inter-locked sleep here.
+	 */
+	SOCK_LOCK(so);
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		tsleep(&so->so_timeo, PSOCK, "nbcon", 2 * hz);
+		msleep(&so->so_timeo, SOCK_MTX(so), PSOCK, "nbcon", 2 * hz);
 		if ((so->so_state & SS_ISCONNECTING) && so->so_error == 0 &&
 			(error = nb_intr(nbp, td->td_proc)) != 0) {
 			so->so_state &= ~SS_ISCONNECTING;
-			splx(s);
+			SOCK_UNLOCK(so);
 			goto bad;
 		}
 	}
 	if (so->so_error) {
 		error = so->so_error;
 		so->so_error = 0;
-		splx(s);
+		SOCK_UNLOCK(so);
 		goto bad;
 	}
-	splx(s);
+	SOCK_UNLOCK(so);
 	return 0;
 bad:
 	smb_nbst_disconnect(nbp->nbp_vc, td);
--- //depot/user/rwatson/sockref/src/sys/nfsclient/nfs_socket.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/nfsclient/nfs_socket.c	2007/04/29 12:41:48
@@ -438,7 +438,7 @@
 	error = soreserve(so, sndreserve, rcvreserve);
 	if (error)
 		goto bad;
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_rcv.sb_flags |= SB_NOINTR;
  	so->so_upcallarg = (caddr_t)nmp;
  	if (so->so_type == SOCK_STREAM)
@@ -446,10 +446,8 @@
  	else	
  		so->so_upcall = nfs_clnt_udp_soupcall;
  	so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_flags |= SB_NOINTR;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 
 	mtx_lock(&nmp->nm_mtx);
 	/* Initialize other non-zero congestion variables */
@@ -538,11 +536,11 @@
 		so = nmp->nm_so;
 		nmp->nm_so = NULL;
 		mtx_unlock(&nmp->nm_mtx);
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
  		so->so_upcallarg = NULL;
  		so->so_upcall = NULL;
  		so->so_rcv.sb_flags &= ~SB_UPCALL;
-		SOCKBUF_UNLOCK(&so->so_rcv);
+		SOCK_UNLOCK(so);
 		soshutdown(so, SHUT_WR);
 		soclose(so);
 	} else
@@ -882,12 +880,12 @@
 nfstcp_readable(struct socket *so, int bytes)
 {
 	int retval;
-	
-	SOCKBUF_LOCK(&so->so_rcv);
+
+	SOCK_LOCK(so);
 	retval = (so->so_rcv.sb_cc >= (bytes) ||
 		  (so->so_rcv.sb_state & SBS_CANTRCVMORE) ||
 		  so->so_error);
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 	return (retval);
 }
 
--- //depot/user/rwatson/sockref/src/sys/nfsserver/nfs_syscalls.c	2006/12/24 14:06:47
+++ //depot/user/rwatson/resock/src/sys/nfsserver/nfs_syscalls.c	2007/04/29 12:41:48
@@ -200,7 +200,7 @@
 	int siz;
 	struct nfssvc_sock *slp;
 	struct socket *so;
-	int error, s;
+	int error;
 
 	NET_ASSERT_GIANT();
 
@@ -265,14 +265,12 @@
 		val = 1;
 		sosetopt(so, &sopt);
 	}
-	SOCKBUF_LOCK(&so->so_rcv);
+	SOCK_LOCK(so);
 	so->so_rcv.sb_flags &= ~SB_NOINTR;
 	so->so_rcv.sb_timeo = 0;
-	SOCKBUF_UNLOCK(&so->so_rcv);
-	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_flags &= ~SB_NOINTR;
 	so->so_snd.sb_timeo = 0;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	SOCK_UNLOCK(so);
 
 	slp = (struct nfssvc_sock *)
 		malloc(sizeof (struct nfssvc_sock), M_NFSSVC,
@@ -288,15 +286,13 @@
 	/*
 	 * XXXRW: Socket locking here?
 	 */
-	s = splnet();
+	SOCK_LOCK(so);
 	so->so_upcallarg = (caddr_t)slp;
 	so->so_upcall = nfsrv_rcv;
-	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_flags |= SB_UPCALL;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	SOCK_UNLOCK(so);
 	slp->ns_flag = (SLP_VALID | SLP_NEEDQ);
 	nfsrv_wakenfsd(slp);
-	splx(s);
 	NFSD_UNLOCK();
 	return (0);
 }
@@ -620,11 +616,11 @@
 		NFSD_UNLOCK();
 		slp->ns_fp = NULL;
 		so = slp->ns_so;
-		SOCKBUF_LOCK(&so->so_rcv);
+		SOCK_LOCK(so);
 		so->so_rcv.sb_flags &= ~SB_UPCALL;
-		SOCKBUF_UNLOCK(&so->so_rcv);
 		so->so_upcall = NULL;
 		so->so_upcallarg = NULL;
+		SOCK_UNLOCK(so);
 		soshutdown(so, SHUT_RDWR);
 		closef(fp, NULL);
 		NFSD_LOCK();
--- //depot/user/rwatson/sockref/src/sys/sys/socketvar.h	2006/08/06 19:49:25
+++ //depot/user/rwatson/resock/src/sys/sys/socketvar.h	2006/11/30 15:30:26
@@ -50,8 +50,6 @@
  * Locking key to struct socket:
  * (a) constant after allocation, no locking required.
  * (b) locked by SOCK_LOCK(so).
- * (c) locked by SOCKBUF_LOCK(&so->so_rcv).
- * (d) locked by SOCKBUF_LOCK(&so->so_snd).
  * (e) locked by ACCEPT_LOCK().
  * (f) not locked since integer reads/writes are atomic.
  * (g) used only as a sleep/wakeup address, no value.
@@ -89,28 +87,29 @@
 	u_short	so_error;		/* (f) error affecting connection */
 	struct	sigio *so_sigio;	/* [sg] information for async I/O or
 					   out of band data (SIGURG) */
-	u_long	so_oobmark;		/* (c) chars to oob mark */
+	u_long	so_oobmark;		/* (b) chars to oob mark */
 	TAILQ_HEAD(, aiocblist) so_aiojobq; /* AIO ops waiting on socket */
+	struct	mtx so_mtx;		/* (a) socket mutex */
 /*
  * Variables for socket buffering.
  */
 	struct sockbuf {
 		struct	selinfo sb_sel;	/* process selecting read/write */
-		struct	mtx sb_mtx;	/* sockbuf lock */
-		short	sb_state;	/* (c/d) socket state on sockbuf */
+		struct	mtx *sb_somtxp;	/* (a) pointer to socket mutex */
+		short	sb_state;	/* (b) socket state on sockbuf */
 #define	sb_startzero	sb_mb
-		struct	mbuf *sb_mb;	/* (c/d) the mbuf chain */
-		struct	mbuf *sb_mbtail; /* (c/d) the last mbuf in the chain */
+		struct	mbuf *sb_mb;	/* (b) the mbuf chain */
+		struct	mbuf *sb_mbtail; /* (b) the last mbuf in the chain */
 		struct	mbuf *sb_lastrecord;	/* (c/d) first mbuf of last
 						 * record in socket buffer */
-		u_int	sb_cc;		/* (c/d) actual chars in buffer */
-		u_int	sb_hiwat;	/* (c/d) max actual char count */
-		u_int	sb_mbcnt;	/* (c/d) chars of mbufs used */
-		u_int	sb_mbmax;	/* (c/d) max chars of mbufs to use */
-		u_int	sb_ctl;		/* (c/d) non-data chars in buffer */
-		int	sb_lowat;	/* (c/d) low water mark */
-		int	sb_timeo;	/* (c/d) timeout for read/write */
-		short	sb_flags;	/* (c/d) flags, see below */
+		u_int	sb_cc;		/* (b) actual chars in buffer */
+		u_int	sb_hiwat;	/* (b) max actual char count */
+		u_int	sb_mbcnt;	/* (b) chars of mbufs used */
+		u_int	sb_mbmax;	/* (b) max chars of mbufs to use */
+		u_int	sb_ctl;		/* (b) non-data chars in buffer */
+		int	sb_lowat;	/* (b) low water mark */
+		int	sb_timeo;	/* (b) timeout for read/write */
+		short	sb_flags;	/* (b) flags, see below */
 	} so_rcv, so_snd;
 /*
  * Constants for sb_flags field of struct sockbuf.
@@ -164,29 +163,47 @@
 #define	ACCEPT_UNLOCK()			mtx_unlock(&accept_mtx)
 
 /*
- * Per-socket buffer mutex used to protect most fields in the socket
- * buffer.
+ * Per-socket mutex: we use a single mutex to cover the entire socket,
+ * including both socket buffers.  In earlier versions of this code, we used
+ * separate send and receive mutexes, which allowed greater parallelism but
+ * also resulted in substantial locking complexity and overhead.  For now,
+ * return to a single mutex to reduce locking costs.  Back-pointers from the
+ * socket buffer allow the lock to be frobbed when only a socket buffer
+ * reference is available.
  */
-#define	SOCKBUF_MTX(_sb)		(&(_sb)->sb_mtx)
-#define	SOCKBUF_LOCK_INIT(_sb, _name) \
-	mtx_init(SOCKBUF_MTX(_sb), _name, NULL, MTX_DEF)
-#define	SOCKBUF_LOCK_DESTROY(_sb)	mtx_destroy(SOCKBUF_MTX(_sb))
-#define	SOCKBUF_LOCK(_sb)		mtx_lock(SOCKBUF_MTX(_sb))
-#define	SOCKBUF_OWNED(_sb)		mtx_owned(SOCKBUF_MTX(_sb))
-#define	SOCKBUF_UNLOCK(_sb)		mtx_unlock(SOCKBUF_MTX(_sb))
-#define	SOCKBUF_LOCK_ASSERT(_sb)	mtx_assert(SOCKBUF_MTX(_sb), MA_OWNED)
-#define	SOCKBUF_UNLOCK_ASSERT(_sb)	mtx_assert(SOCKBUF_MTX(_sb), MA_NOTOWNED)
+#define	SOCK_MTX(_so)			(&(_so)->so_mtx)
+#define	SOCK_LOCK_INIT(_so) do {					\
+	mtx_init(SOCK_MTX(_so), "socket", NULL, MTX_DEF | MTX_RECURSE);	\
+	(_so)->so_rcv.sb_somtxp = SOCK_MTX(_so);			\
+	(_so)->so_snd.sb_somtxp = SOCK_MTX(_so);			\
+} while (0)
+#define	SOCK_LOCK_DESTROY(_so)		mtx_destroy(SOCK_MTX(_so))
+#define	SOCK_LOCK(_so)			mtx_lock(SOCK_MTX(_so))
+#define	SOCK_UNLOCK(_so)		mtx_unlock(SOCK_MTX(_so))
+#define	SOCK_OWNED(_so)			mtx_owned(SOCK_MTX(_so))
+#define	SOCK_LOCK_ASSERT(_so)		mtx_assert(SOCK_MTX(_so), MA_OWNED)
+#define	SOCK_UNLOCK_ASSERT(_so)		mtx_assert(SOCK_MTX(_so), MA_NOTOWNED)
+
+#define	SOCKBUF_MTX(_sb)		((_sb)->sb_somtxp)
+#define	SOCKBUF_LOCK(_sb) do {						\
+	if (SOCKBUF_MTX(_sb) != NULL)					\
+		mtx_lock(SOCKBUF_MTX(_sb));				\
+} while (0)
+
+#define	SOCKBUF_UNLOCK(_sb) do {					\
+	if (SOCKBUF_MTX(_sb) != NULL)					\
+		mtx_unlock(SOCKBUF_MTX(_sb));				\
+} while (0)
+
+#define	SOCKBUF_LOCK_ASSERT(_sb) do {					\
+	if (SOCKBUF_MTX(_sb) != NULL)					\
+		mtx_assert(SOCKBUF_MTX(_sb), MA_OWNED);			\
+} while (0)
 
-/*
- * Per-socket mutex: we reuse the receive socket buffer mutex for space
- * efficiency.  This decision should probably be revisited as we optimize
- * locking for the socket code.
- */
-#define	SOCK_MTX(_so)			SOCKBUF_MTX(&(_so)->so_rcv)
-#define	SOCK_LOCK(_so)			SOCKBUF_LOCK(&(_so)->so_rcv)
-#define	SOCK_OWNED(_so)			SOCKBUF_OWNED(&(_so)->so_rcv)
-#define	SOCK_UNLOCK(_so)		SOCKBUF_UNLOCK(&(_so)->so_rcv)
-#define	SOCK_LOCK_ASSERT(_so)		SOCKBUF_LOCK_ASSERT(&(_so)->so_rcv)
+#define	SOCKBUF_UNLOCK_ASSERT(_sb) do {					\
+	if (SOCKBUF_MTX(_sb) != NULL)					\
+		mtx_assert(SOCKBUF_MTX(_sb), MA_NOTOWNED);		\
+} while (0)
 
 /*
  * Socket state bits.
@@ -376,37 +393,33 @@
 } while(0)
 
 /*
- * In sorwakeup() and sowwakeup(), acquire the socket buffer lock to
- * avoid a non-atomic test-and-wakeup.  However, sowakeup is
- * responsible for releasing the lock if it is called.  We unlock only
- * if we don't call into sowakeup.  If any code is introduced that
- * directly invokes the underlying sowakeup() primitives, it must
- * maintain the same semantics.
+ * In sorwakeup() and sowwakeup(), acquire the socket lock to avoid a
+ * non-atomic test-and-wakeup.
+ *
+ * XXXRW: The _locked() macros used to release the lock on return.
  */
 #define	sorwakeup_locked(so) do {					\
-	SOCKBUF_LOCK_ASSERT(&(so)->so_rcv);				\
+	SOCK_LOCK_ASSERT(so);						\
 	if (sb_notify(&(so)->so_rcv))					\
 		sowakeup((so), &(so)->so_rcv);	 			\
-	else								\
-		SOCKBUF_UNLOCK(&(so)->so_rcv);				\
 } while (0)
 
 #define	sorwakeup(so) do {						\
-	SOCKBUF_LOCK(&(so)->so_rcv);					\
+	SOCK_LOCK(so);							\
 	sorwakeup_locked(so);						\
+	SOCK_UNLOCK(so);						\
 } while (0)
 
 #define	sowwakeup_locked(so) do {					\
-	SOCKBUF_LOCK_ASSERT(&(so)->so_snd);				\
+	SOCK_LOCK_ASSERT(so);						\
 	if (sb_notify(&(so)->so_snd))					\
 		sowakeup((so), &(so)->so_snd); 				\
-	else								\
-		SOCKBUF_UNLOCK(&(so)->so_snd);				\
 } while (0)
 
 #define	sowwakeup(so) do {						\
-	SOCKBUF_LOCK(&(so)->so_snd);					\
+	SOCK_LOCK(so);							\
 	sowwakeup_locked(so);						\
+	SOCK_UNLOCK(so);						\
 } while (0)
 
 /*
@@ -539,6 +552,7 @@
 	    struct uio *uio, struct mbuf **mp0, struct mbuf **controlp,
 	    int *flagsp);
 int	soreserve(struct socket *so, u_long sndcc, u_long rcvcc);
+void	sorflush_locked(struct socket *so);
 void	sorflush(struct socket *so);
 int	sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	    struct mbuf *top, struct mbuf *control, int flags,
