--- //depot/vendor/freebsd/src/sys/conf/options	2007/03/22 16:12:06
+++ //depot/user/attilio/attilio_smpng/conf/options	2007/03/22 16:53:25
@@ -59,6 +59,7 @@
 
 # Miscellaneous options.
 ADAPTIVE_GIANT	opt_adaptive_mutexes.h
+ADAPTIVE_SX	opt_adaptive_mutexes.h
 NO_ADAPTIVE_MUTEXES	opt_adaptive_mutexes.h
 ALQ
 AUDIT		opt_global.h
@@ -555,6 +556,7 @@
 REGRESSION		opt_global.h
 RESTARTABLE_PANICS	opt_global.h
 RWLOCK_NOINLINE		opt_global.h
+SXLOCK_NOINLINE		opt_global.h
 VFS_BIO_DEBUG		opt_global.h
 
 # These are VM related options
--- //depot/vendor/freebsd/src/sys/dev/acpica/acpi_ec.c	2007/03/20 01:02:34
+++ //depot/user/attilio/attilio_smpng/dev/acpica/acpi_ec.c	2007/03/21 14:26:34
@@ -142,6 +142,7 @@
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/sx.h>
--- //depot/vendor/freebsd/src/sys/dev/led/led.c	2005/03/07 11:10:19
+++ //depot/user/attilio/attilio_smpng/dev/led/led.c	2006/11/04 01:33:58
@@ -15,6 +15,7 @@
 #include <sys/conf.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/ctype.h>
 #include <sys/sbuf.h>
--- //depot/vendor/freebsd/src/sys/dev/mxge/if_mxge.c	2007/02/23 12:24:01
+++ //depot/user/attilio/attilio_smpng/dev/mxge/if_mxge.c	2007/03/04 00:12:48
@@ -40,6 +40,7 @@
 #include <sys/firmware.h>
 #include <sys/endian.h>
 #include <sys/sockio.h>
+#include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/kdb.h>
--- //depot/vendor/freebsd/src/sys/dev/sound/pcm/sndstat.c	2007/03/16 17:15:52
+++ //depot/user/attilio/attilio_smpng/dev/sound/pcm/sndstat.c	2007/03/18 15:17:54
@@ -27,6 +27,7 @@
 #include <dev/sound/pcm/sound.h>
 #include <dev/sound/pcm/vchan.h>
 #ifdef	USING_MUTEX
+#include <sys/lock.h>
 #include <sys/sx.h>
 #endif
 
--- //depot/vendor/freebsd/src/sys/dev/usb/if_aue.c	2007/02/03 13:57:44
+++ //depot/user/attilio/attilio_smpng/dev/usb/if_aue.c	2007/02/06 01:28:29
@@ -71,6 +71,7 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/sockio.h>
+#include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
--- //depot/vendor/freebsd/src/sys/dev/usb/if_axe.c	2006/10/19 01:18:01
+++ //depot/user/attilio/attilio_smpng/dev/usb/if_axe.c	2006/11/04 01:33:58
@@ -70,6 +70,7 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/sockio.h>
+#include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
--- //depot/vendor/freebsd/src/sys/gnu/fs/xfs/FreeBSD/support/mrlock.c	2005/12/12 01:05:49
+++ //depot/user/attilio/attilio_smpng/gnu/fs/xfs/FreeBSD/support/mrlock.c	2007/03/22 16:03:37
@@ -1,49 +1,14 @@
 #include <sys/param.h>
+#include <machine/pcpu.h>
+#include <support/debug.h>
 #include <support/mrlock.h>
 
-void
-_sx_xfs_destroy(struct sx *sx)
+int
+ismrlocked(mrlock_t *mrp, int type)
 {
-	if (sx->sx_cnt == -1)
-		sx_xunlock(sx);
-	sx_destroy(sx);
-}
 
-void
-_sx_xfs_lock(struct sx *sx, int type, const char *file, int line)
-{
-	if (type == MR_ACCESS)
-		_sx_slock(sx, file, line);
-	else if (type == MR_UPDATE)
-		_sx_sunlock(sx, file, line);
-	else
-		panic("Invalid lock type passed");
+	sx_assert(mrp, SX_LOCKED);
+	if (type == MR_UPDATE)
+		return sx_xlocked(mrp);
+	return 1;
 }
-
-
-void
-_sx_xfs_unlock(struct sx *sx, const char *file, int line)
-{
-	if (_sx_xfs_xowned(sx))
-		_sx_xunlock(sx, file, line);
-	else if (_sx_xfs_sowned(sx))
-		_sx_sunlock(sx, file, line);
-	else
-		panic("lock is not locked");
-}
-
-int
-ismrlocked(mrlock_t *mrp, int type)
-{	
-	if (type == MR_ACCESS)
-		return _sx_xfs_sowned(mrp); /* Read lock */
-	else if (type == MR_UPDATE)
-		return _sx_xfs_xowned(mrp); /* Write lock */
-	else if (type == (MR_UPDATE | MR_ACCESS))
-		return  _sx_xfs_sowned(mrp) ||
-		        _sx_xfs_xowned(mrp); /* Any type of lock held */
-	return (mrp->sx_shrd_wcnt > 0 || mrp->sx_excl_wcnt > 0);
-}
-
-
-
--- //depot/vendor/freebsd/src/sys/gnu/fs/xfs/FreeBSD/support/mrlock.h	2005/12/12 01:05:49
+++ //depot/user/attilio/attilio_smpng/gnu/fs/xfs/FreeBSD/support/mrlock.h	2007/03/16 19:26:02
@@ -4,62 +4,38 @@
 #include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
-#include <sys/mutex.h>
 #include <sys/sx.h>
 
-#include <support/debug.h>
-
 /*
  * Implement mrlocks on FreeBSD that work for XFS.
- * Use FreeBSD sx lock and add necessary functions
- * if additional functionality is requested
+ * Map mrlock functions to corresponding equivalents in
+ * sx.
  */
 typedef struct sx mrlock_t;
 
 #define MR_ACCESS	1
 #define MR_UPDATE	2
 
-/* 
+/*
  * Compatibility defines, not really used
  */
 #define MRLOCK_BARRIER		0x1
 #define MRLOCK_ALLOW_EQUAL_PRI	0x8
 
-/*
- * mraccessf/mrupdatef take flags to be passed in while sleeping;
- * only PLTWAIT is currently supported.
- */
-#define mrinit(lock, name)	sx_init(lock, name)
 #define mrlock_init(lock, type, name, seq) sx_init(lock, name)
-#define mrfree(lock)		_sx_xfs_destroy(lock)
-#define	mraccessf(lock, f)	sx_slock(lock)
-#define	mrupdatef(lock, f)	sx_xlock(lock)
-#define mraccunlock(lock)	sx_sunlock(lock)
 #define mrtryaccess(lock)	sx_try_slock(lock)
 #define mrtryupdate(lock)	sx_try_xlock(lock)
-#define mraccess(mrp)		mraccessf(mrp, 0)
-#define mrupdate(mrp)		mrupdatef(mrp, 0)
-#define mrislocked_access(lock)	_sx_xfs_xowned(lock)
-#define mrislocked_update(lock)	_sx_xfs_sowned(lock)
-#define mrtrypromote(lock)	sx_try_upgrade(lock)
+#define mraccess(lock)		sx_slock(lock)
+#define mrupdate(lock)		sx_xlock(lock)
 #define mrdemote(lock)		sx_downgrade(lock)
+#define mrunlock(lock)		sx_unlock(lock)
 
-int	ismrlocked(mrlock_t *, int);
-void	_sx_xfs_lock(struct sx *sx, int type, const char *file, int line);
-void	_sx_xfs_unlock(struct sx *sx, const char *file, int line);
-void	_sx_xfs_destroy(struct sx *sx);
-#define _sx_xfs_xowned(lock) ((lock)->sx_cnt < 0)
-#define _sx_xfs_sowned(lock) ((lock)->sx_cnt > 0)
+#define mrfree(lock) do {		\
+	if (sx_xlocked(lock))		\
+		sx_xunlock(lock);	\
+	sx_destroy(lock);		\
+} while (0)
 
-/*
- * Functions, not implemented in FreeBSD 
- */
-#define mrunlock(lock) \
-        _sx_xfs_unlock(lock, __FILE__, __LINE__)
-
-#define	mrlock(lock, type, flags) \
-        _sx_xfs_lock(lock, type, __FILE__, __LINE__)
-
-
+int ismrlocked(mrlock_t *mrp, int type);
 
 #endif /* __XFS_SUPPORT_MRLOCK_H__ */
--- //depot/vendor/freebsd/src/sys/i386/acpica/acpi_machdep.c	2006/08/11 19:25:48
+++ //depot/user/attilio/attilio_smpng/i386/acpica/acpi_machdep.c	2006/12/05 20:24:17
@@ -29,6 +29,7 @@
 
 #include <sys/param.h>
 #include <sys/bus.h>
+#include <sys/condvar.h>
 #include <sys/conf.h>
 #include <sys/fcntl.h>
 #include <sys/kernel.h>
--- //depot/vendor/freebsd/src/sys/kern/init_main.c	2007/01/23 08:52:34
+++ //depot/user/attilio/attilio_smpng/kern/init_main.c	2007/03/10 00:07:51
@@ -94,7 +94,7 @@
 static struct session session0;
 static struct pgrp pgrp0;
 struct	proc proc0;
-struct	thread thread0 __aligned(8);
+struct	thread thread0 __aligned(16);
 struct	vmspace vmspace0;
 struct	proc *initproc;
 
--- //depot/vendor/freebsd/src/sys/kern/kern_sx.c	2007/03/21 21:22:46
+++ //depot/user/attilio/attilio_smpng/kern/kern_sx.c	2007/03/22 15:55:21
@@ -1,5 +1,7 @@
 /*-
- * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>.  All rights reserved.
+ * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
+ * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -25,33 +27,54 @@
  * DAMAGE.
  */
 
-/*
- * Shared/exclusive locks.  This implementation assures deterministic lock
- * granting behavior, so that slocks and xlocks are interleaved.
- *
- * Priority propagation will not generally raise the priority of lock holders,
- * so should not be relied upon in combination with sx locks.
- */
+#include "opt_adaptive_mutexes.h"
+#include "opt_ddb.h"
 
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/kern/kern_sx.c,v 1.39 2007/03/21 21:20:50 jhb Exp $");
 
-#include "opt_ddb.h"
-
 #include <sys/param.h>
-#include <sys/systm.h>
+#ifdef DDB
+#include <sys/condvar.h>
+#endif
 #include <sys/ktr.h>
-#include <sys/linker_set.h>
-#include <sys/condvar.h>
 #include <sys/lock.h>
+#include <sys/lock_profile.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/sleepqueue.h>
 #include <sys/sx.h>
-#include <sys/lock_profile.h>
+#include <sys/systm.h>
+
+#ifdef ADAPTIVE_SX
+#include <machine/cpu.h>
+#endif
 
 #ifdef DDB
 #include <ddb/ddb.h>
+#endif
+
+#if !defined(SMP) && defined(ADAPTIVE_SX)
+#error "You must have SMP before to enable ADAPTIVE_SX option"
+#endif
 
+#ifndef INVARIANTS
+#define	_sx_assert(sx, what, file, line)
+#endif
+
+/*
+ * Returns if the lock is recursed. It assumes a check on curthread holding
+ * the lock.
+ */
+#define	sx_recursed(sx)		((sx)->sx_recurse != 0)
+
+/*
+ * Currently, we keep track of the exclusive holder for debugging purposes.
+ */
+#define	sx_xholder(sx)							\
+	((sx)->sx_lock & SX_SHARED_BIT ? NULL :				\
+	(struct thread *)SX_HOLDER((sx)->sx_lock))
+
+#ifdef DDB
 static void	db_show_sx(struct lock_object *lock);
 #endif
 static void	lock_sx(struct lock_object *lock, int how);
@@ -67,10 +90,6 @@
 	.lc_unlock = unlock_sx,
 };
 
-#ifndef INVARIANTS
-#define	_sx_assert(sx, what, file, line)
-#endif
-
 void
 lock_sx(struct lock_object *lock, int how)
 {
@@ -89,7 +108,7 @@
 	struct sx *sx;
 
 	sx = (struct sx *)lock;
-	sx_assert(sx, SX_LOCKED | LA_NOTRECURSED);
+	sx_assert(sx, SX_LOCKED | SX_NOTRECURSED);
 	if (sx_xlocked(sx)) {
 		sx_xunlock(sx);
 		return (1);
@@ -111,255 +130,599 @@
 sx_init(struct sx *sx, const char *description)
 {
 
-	sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx);
-	sx->sx_cnt = 0;
-	cv_init(&sx->sx_shrd_cv, description);
-	sx->sx_shrd_wcnt = 0;
-	cv_init(&sx->sx_excl_cv, description);
-	sx->sx_excl_wcnt = 0;
-	sx->sx_xholder = NULL;
+	sx->sx_desc = description;
+	sx->sx_flags = 0;
+	sx->sx_lock = SX_UNHELD;
+	sx->sx_recurse = 0;
 	lock_profile_object_init(&sx->lock_object, &lock_class_sx, description);
 	lock_init(&sx->lock_object, &lock_class_sx, description, NULL,
 	    LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE);
 }
 
 void
+sx_init_flags(struct sx *sx, const char *description, int opts)
+{
+	int flags;
+
+	flags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE;
+	if (opts & SX_DUPOK)
+		flags |= LO_DUPOK;
+	if (opts & SX_NOPROFILE)
+		flags |= LO_NOPROFILE;
+	if (!(opts & SX_NOWITNESS))
+		flags |= LO_WITNESS;
+	if (opts & SX_QUIET)
+		flags |= LO_QUIET;
+
+	sx->sx_desc = description;
+	sx->sx_flags = opts & SX_ADAPTIVESPIN;
+	sx->sx_lock = SX_UNHELD;
+	sx->sx_recurse = 0;
+	lock_profile_object_init(&sx->lock_object, &lock_class_sx, description);
+	lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags);
+}
+
+void
 sx_destroy(struct sx *sx)
 {
 
-	KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt ==
-	    0), ("%s (%s): holders or waiters\n", __func__,
-	    sx->lock_object.lo_name));
-
-	sx->sx_lock = NULL;
-	cv_destroy(&sx->sx_shrd_cv);
-	cv_destroy(&sx->sx_excl_cv);
-	
+	KASSERT(sx->sx_lock == SX_UNHELD, ("sx lock still held"));
+	KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
 	lock_profile_object_destroy(&sx->lock_object);
 	lock_destroy(&sx->lock_object);
 }
 
 void
-_sx_slock(struct sx *sx, const char *file, int line)
+_sx_xlock(struct sx *sx, const char *file, int line)
 {
-	uint64_t waittime = 0;
-	int contested = 0;
 
-	mtx_lock(sx->sx_lock);
-	KASSERT(sx->sx_xholder != curthread,
-	    ("%s (%s): slock while xlock is held @ %s:%d\n", __func__,
-	    sx->lock_object.lo_name, file, line));
-	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line);
+	MPASS(curthread != NULL);
+	witness_check(sx);
+	__sx_xlock(sx, curthread, file, line);
+	LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, 0, file, line);
+	WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
+	curthread->td_locks++;
+}
 
-	/*
-	 * Loop in case we lose the race for lock acquisition.
-	 */
-	while (sx->sx_cnt < 0) {
-		sx->sx_shrd_wcnt++;
-		lock_profile_obtain_lock_failed(&sx->lock_object, &contested, &waittime);
-		cv_wait(&sx->sx_shrd_cv, sx->sx_lock);
-		sx->sx_shrd_wcnt--;
-	}
+void
+_sx_xunlock(struct sx *sx, const char *file, int line)
+{
 
-	/* Acquire a shared lock. */
-	sx->sx_cnt++;
+	MPASS(curthread != NULL);
+	_sx_assert(sx, SX_XLOCKED, file, line);
+	curthread->td_locks--;
+	WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
+	LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, 0, file, line);
+	lock_profile_release_lock(&sx->lock_object);
+	__sx_xunlock(sx, curthread, file, line);
+}
 
-        if (sx->sx_cnt == 1)
-		lock_profile_obtain_lock_success(&sx->lock_object, contested, waittime, file, line);
+void
+_sx_slock(struct sx *sx, const char *file, int line)
+{
 
+	witness_check_shared(sx);
+	__sx_slock(sx, file, line);
 	LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
 	WITNESS_LOCK(&sx->lock_object, 0, file, line);
 	curthread->td_locks++;
+}
+
+void
+_sx_sunlock(struct sx *sx, const char *file, int line)
+{
+
+	_sx_assert(sx, SX_SLOCKED, file, line);
+	curthread->td_locks--;
+	WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
+	LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
+	lock_profile_release_lock(&sx->lock_object);
+	__sx_sunlock(sx, file, line);
+}
+
+/*
+ * Try to do a non-blocking sx_xlock.  Return 1 if succeed, 0 otherwise.
+ */
+int
+_sx_try_xlock(struct sx *sx, const char *file, int line)
+{
+	uintptr_t tid;
+
+	MPASS(curthread != NULL);
 
-	mtx_unlock(sx->sx_lock);
+	tid = (uintptr_t)curthread;
+	if (atomic_cmpset_acq_ptr(&sx->sx_lock, SX_UNHELD, tid)) {
+		LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, 1, file, line);
+		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
+		    file, line);
+		curthread->td_locks++;
+		return (1);
+	}
+
+	LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, 0, file, line);
+	return (0);
 }
 
+/*
+ * Try to do a non-blocking sx_slock.  Return 1 if succeed, 0 otherwise.
+ */
 int
 _sx_try_slock(struct sx *sx, const char *file, int line)
 {
+	uintptr_t x;
 
-	mtx_lock(sx->sx_lock);
-	if (sx->sx_cnt >= 0) {
-		sx->sx_cnt++;
+	x = sx->sx_lock;
+	if ((x & SX_SHARED_BIT) && atomic_cmpset_acq_ptr(&sx->sx_lock, x,
+	    x + SX_ONE_SHARER)) {
 		LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
 		WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
 		curthread->td_locks++;
-		mtx_unlock(sx->sx_lock);
 		return (1);
-	} else {
-		LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
-		mtx_unlock(sx->sx_lock);
-		return (0);
 	}
+
+	LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
+	return (0);
 }
 
-void
-_sx_xlock(struct sx *sx, const char *file, int line)
+/*
+ * Try to do a non-blocking upgrade from a shared lock to an exclusive lock.
+ * This will only succeed if this thread holds a single shared lock.
+ * Return 1 if if the upgrade succeed, 0 otherwise.
+ */
+int
+_sx_try_upgrade(struct sx *sx, const char *file, int line)
 {
-	int contested = 0;
-	uint64_t waittime = 0;
+	uintptr_t tid, x;
+
+	_sx_assert(sx, SX_SLOCKED, file, line);
 
-	mtx_lock(sx->sx_lock);
+	tid = (uintptr_t)curthread;
 
 	/*
-	 * With sx locks, we're absolutely not permitted to recurse on
-	 * xlocks, as it is fatal (deadlock). Normally, recursion is handled
-	 * by WITNESS, but as it is not semantically correct to hold the
-	 * xlock while in here, we consider it API abuse and put it under
-	 * INVARIANTS.
+	 * Try to switch from one sharer to an exclusive lock without
+	 * lock the sleepqueue (assume that there aren't any exclusive waiter).
 	 */
-	KASSERT(sx->sx_xholder != curthread,
-	    ("%s (%s): xlock already held @ %s:%d", __func__,
-	    sx->lock_object.lo_name, file, line));
-	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
-	    line);
-
-	/* Loop in case we lose the race for lock acquisition. */
-	while (sx->sx_cnt != 0) {
-		sx->sx_excl_wcnt++;
-		lock_profile_obtain_lock_failed(&sx->lock_object, &contested, &waittime);
-		cv_wait(&sx->sx_excl_cv, sx->sx_lock);
-		sx->sx_excl_wcnt--;
+	if (!(sx->sx_lock & SX_EXCLUSIVE_WAITERS_BIT) &&
+	    atomic_cmpset_acq_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1), tid)) {
+		LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, 1, file, line);
+		WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
+		    file, line);
+		return (1);
 	}
 
-	MPASS(sx->sx_cnt == 0);
+	sleepq_lock(&sx->lock_object);
 
-	/* Acquire an exclusive lock. */
-	sx->sx_cnt--;
-	sx->sx_xholder = curthread;
+	/*
+	 * Since we might have SX_EXCLUSIVE_WAITERS_BIT set, mantain it (if
+	 * really present) in order to preserve a consistent state.  Than,
+	 * if upgrade succeeds return true, false otherwise.
+	 */
+	x = sx->sx_lock & SX_EXCLUSIVE_WAITERS_BIT;
+	if (atomic_cmpset_acq_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x,
+	    tid | x)) {
+		sleepq_release(&sx->lock_object);
+		LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, 1, file, line);
+		WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
+		    file, line);
+		return (1);
+	}
+	sleepq_release(&sx->lock_object);
 
-	lock_profile_obtain_lock_success(&sx->lock_object, contested, waittime, file, line);
-	LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, 0, file, line);
-	WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
-	curthread->td_locks++;
-
-	mtx_unlock(sx->sx_lock);
+	LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, 0, file, line);
+	return (0);
 }
 
-int
-_sx_try_xlock(struct sx *sx, const char *file, int line)
+/*
+ * Downgrade an exclusive lock into a shared lock.
+ */
+void
+_sx_downgrade(struct sx *sx, const char *file, int line)
 {
+	uintptr_t tid, x;
+
+	_sx_assert(sx, SX_XLOCKED, file, line);
+	WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
+
+	tid = (uintptr_t)curthread;
 
-	mtx_lock(sx->sx_lock);
-	if (sx->sx_cnt == 0) {
-		sx->sx_cnt--;
-		sx->sx_xholder = curthread;
-		LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, 1, file, line);
-		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file,
-		    line);
-		curthread->td_locks++;
-		mtx_unlock(sx->sx_lock);
-		return (1);
-	} else {
-		LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, 0, file, line);
-		mtx_unlock(sx->sx_lock);
-		return (0);
+	/*
+	 * Try to switch from an exclusive lock to one sharer without lock the
+	 * sleepqueue (assume that there aren't any waiter).
+	 */
+	if (atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_SHARERS_LOCK(1))) {
+		LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
+		return;
 	}
+
+	sleepq_lock(&sx->lock_object);
+
+	/*
+	 * Since we could have SX_EXCLUSIVE_WAITERS_BIT set, mantain it (if
+	 * really present) in order to preserve a consistent state.  Than,
+	 * wake up eventual shared waiters here and downgrade the lock.
+	 */
+	x = sx->sx_lock;
+	atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | (x &
+	    SX_EXCLUSIVE_WAITERS_BIT));
+	if (x & SX_SHARED_WAITERS_BIT)
+		sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 1);
+	else
+		sleepq_release(&sx->lock_object);
+
+	LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
 }
 
+/*
+ * This function rappresents the so-called 'hard case' for sx_xlock operation.
+ * All 'easy case' failures are redirected to this.  Note that ideally
+ * this might be a static function, but it needs to be accessible at least
+ * from sx.h.
+ */
 void
-_sx_sunlock(struct sx *sx, const char *file, int line)
+_sx_xlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 {
-	_sx_assert(sx, SX_SLOCKED, file, line);
-	mtx_lock(sx->sx_lock);
+#ifdef ADAPTIVE_SX
+	struct thread *owner;
+#endif
+	uint64_t waittime;
+	uintptr_t x;
+	unsigned gtcnt;
+	int contested;
+
+	waittime = 0;
+	gtcnt = 0;
+	contested = 0;
+	if (sx_xlocked(sx)) {
+		sx->sx_recurse++;
+		atomic_set_ptr(&sx->sx_lock, SX_RECURSED_BIT);
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
+		return;
+	}
+
+	while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_UNHELD, tid)) {
+#ifdef ADAPTIVE_SX
+		x = sx->sx_lock;
+		if (!(x & SX_SHARED_BIT) && (sx->sx_flags & SX_ADAPTIVESPIN)) {
+			owner = (struct thread *)SX_HOLDER(x);
+			x = SX_HOLDER(x);
+			if (TD_IS_RUNNING(owner)) {
+				while (mtx_owned(&Giant)) {
+					gtcnt++;
+					mtx_unlock(&Giant);
+				}
+				while (SX_HOLDER(sx->sx_lock) == x &&
+				    TD_IS_RUNNING(owner))
+					cpu_spinwait();
+				continue;
+			}
+		}
+#endif
+		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
+		    &waittime);
+
+		sleepq_lock(&sx->lock_object);
+		x = sx->sx_lock;
+
+#ifdef ADAPTIVE_SX
+		owner = (struct thread *)SX_HOLDER(x);
+		if (!(x & SX_SHARED_BIT) && (sx->sx_flags & SX_ADAPTIVESPIN) &&
+		    TD_IS_RUNNING(owner)) {
+			sleepq_release(&sx->lock_object);
+			cpu_spinwait();
+			continue;
+		}
+#endif
+
+		/*
+		 * If the lock was released while spinning on the sleepqueue
+		 * chain lock, try again.
+		 */
+		if (x == SX_UNHELD) {
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
 
-	curthread->td_locks--;
-	WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
+		/*
+		 * Since we assume a share locked variable, only
+		 * SX_EXCLUSIVE_WAITERS_BIT could be set, so, if a pending
+		 * queue of exclusive sleepers is left on sx_xunlock, simply
+		 * reset the lock to held state.
+		 */
+		if (x == (SX_UNHELD | SX_EXCLUSIVE_WAITERS_BIT)) {
+			if (atomic_cmpset_acq_ptr(&sx->sx_lock, SX_UNHELD |
+			    SX_EXCLUSIVE_WAITERS_BIT, tid |
+			    SX_EXCLUSIVE_WAITERS_BIT)) {
+				sleepq_release(&sx->lock_object);
+				break;
+			}
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
 
-	/* Release. */
-	sx->sx_cnt--;
+		/*
+		 * Try to set the SX_EXCLUSIVE_WAITERS_BIT.  If we fail,
+		 * than loop back and retry.
+		 */
+		if (!(x & SX_EXCLUSIVE_WAITERS_BIT) &&
+		    !atomic_cmpset_ptr(&sx->sx_lock, x, x |
+		    SX_EXCLUSIVE_WAITERS_BIT)) {
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
 
-	if (sx->sx_cnt == 0) {
-		lock_profile_release_lock(&sx->lock_object);
-	}
+		/*
+		 * Since we have been unable to acquire the exclusive lock,
+		 * we have to sleep.
+		 */
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p blocking on sleepqueue",
+			    __func__, sx);
 
-	/*
-	 * If we just released the last shared lock, wake any waiters up, giving
-	 * exclusive lockers precedence.  In order to make sure that exclusive
-	 * lockers won't be blocked forever, don't wake shared lock waiters if
-	 * there are exclusive lock waiters.
-	 */
-	if (sx->sx_excl_wcnt > 0) {
-		if (sx->sx_cnt == 0)
-			cv_signal(&sx->sx_excl_cv);
-	} else if (sx->sx_shrd_wcnt > 0)
-		cv_broadcast(&sx->sx_shrd_cv);
+		while (mtx_owned(&Giant)) {
+			gtcnt++;
+			mtx_unlock(&Giant);
+		}
+		sleepq_add(&sx->lock_object, NULL, sx->sx_desc, SLEEPQ_SX, 0);
+		sleepq_wait(&sx->lock_object);
 
-	LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p resuming from sleepqueue",
+			    __func__, sx);
+	}
 
-	mtx_unlock(sx->sx_lock);
+	lock_profile_obtain_lock_success(&sx->lock_object, contested, waittime,
+	    file, line);
+	while (gtcnt) {
+		mtx_lock(&Giant);
+		gtcnt--;
+	}
 }
 
+/*
+ * This function rappresents the so-called 'hard case' for sx_xunlock operation.
+ * All 'easy case' failures are redirected to this.  Note that ideally
+ * this might be a static function, but it needs to be accessible at least
+ * from sx.h.
+ */
 void
-_sx_xunlock(struct sx *sx, const char *file, int line)
+_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
 {
-	_sx_assert(sx, SX_XLOCKED, file, line);
-	mtx_lock(sx->sx_lock);
-	MPASS(sx->sx_cnt == -1);
+	uintptr_t x;
+	int queue;
 
-	curthread->td_locks--;
-	WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
+	MPASS(sx->sx_lock & (SX_SHARED_WAITERS_BIT | SX_EXCLUSIVE_WAITERS_BIT));
+	MPASS(!(sx->sx_lock & SX_SHARED_BIT));
+	if (sx_xlocked(sx) && sx_recursed(sx)) {
+		if (!(--sx->sx_recurse))
+			atomic_clear_ptr(&sx->sx_lock, SX_RECURSED_BIT);
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
+		return;
+	}
+	if (LOCK_LOG_TEST(&sx->lock_object, 0))
+		CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
 
-	/* Release. */
-	sx->sx_cnt++;
-	sx->sx_xholder = NULL;
+	sleepq_lock(&sx->lock_object);
+	x = SX_UNHELD;
 
 	/*
-	 * Wake up waiters if there are any.  Give precedence to slock waiters.
+	 * The waking up algorithm here is quite simple and, probabilly, not
+	 * ideal:
+	 * It gives precedence to shared waiters (if they are present).  For
+	 * this condition, we have to preserve the state of the exclusive
+	 * waiters flag.
 	 */
-	if (sx->sx_shrd_wcnt > 0)
-		cv_broadcast(&sx->sx_shrd_cv);
-	else if (sx->sx_excl_wcnt > 0)
-		cv_signal(&sx->sx_excl_cv);
+	if (sx->sx_lock & SX_SHARED_WAITERS_BIT) {
+		queue = 1;
+		x |= (sx->sx_lock & SX_EXCLUSIVE_WAITERS_BIT);
+	} else
+		queue = 0;
 
-	LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, 0, file, line);
-
-	lock_profile_release_lock(&sx->lock_object);
-	mtx_unlock(sx->sx_lock);
+	/*
+	 * Wake up all the waiters for the specific queue.
+	 */
+	if (LOCK_LOG_TEST(&sx->lock_object, 0))
+		CTR3(KTR_LOCK, "%s: %p waking up all thread on %s queue",
+		    __func__, sx, queue == 1 ? "shared" : "exclusive");
+	atomic_store_rel_ptr(&sx->sx_lock, x);
+	sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, queue);
 }
 
-int
-_sx_try_upgrade(struct sx *sx, const char *file, int line)
+/*
+ * This function rappresents the so-called 'hard case' for sx_slock operation.
+ * All 'easy case' failures are redirected to this.  Note that ideally
+ * this might be a static function, but it needs to be accessible at least
+ * from sx.h.
+ */
+void
+_sx_slock_hard(struct sx *sx, const char *file, int line)
 {
+#ifdef ADAPTIVE_SX
+	struct thread *owner;
+#endif
+	uint64_t waittime;
+	uintptr_t x;
+	unsigned gtcnt;
+	int contested;
 
-	_sx_assert(sx, SX_SLOCKED, file, line);
-	mtx_lock(sx->sx_lock);
+	waittime = 0;
+	gtcnt = 0;
+	contested = 0;
+	for (;;) {
+		x = sx->sx_lock;
+
+		/*
+		 * If no other thread has an exclusive lock then try to bump up
+		 * the count of sharers.  Since we have to preserve the state
+		 * of SX_EXCLUSIVE_WAITERS_BIT, if we fail to acquire the
+		 * shared lock loop back and retry.
+		 */
+		if (x & SX_SHARED_BIT) {
+			MPASS(!(x & SX_SHARED_WAITERS_BIT));
+			if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
+			    x + SX_ONE_SHARER)) {
+				if (SX_SHARERS(x) == 0)
+					lock_profile_obtain_lock_success(
+					    &sx->lock_object, contested,
+					    waittime, file, line);
+				break;
+			}
+			continue;
+		}
+
+#ifdef ADAPTIVE_SX
+		x = sx->sx_lock;
+		if (!(x & SX_SHARED_BIT) && (sx->sx_flags & SX_ADAPTIVESPIN)) {
+			owner = (struct thread *)SX_HOLDER(x);
+			x = SX_HOLDER(x);
+			if (TD_IS_RUNNING(owner)) {
+				while (mtx_owned(&Giant)) {
+					gtcnt++;
+					mtx_unlock(&Giant);
+				}
+				while (SX_HOLDER(sx->sx_lock) == x &&
+				    TD_IS_RUNNING(owner))
+					cpu_spinwait();
+				continue;
+			}
+		}
+#endif
+
+		/*
+		 * Some other thread alredy has an exclusive lock, so start
+		 * process of blocking.
+		 */
+		sleepq_lock(&sx->lock_object);
+		x = sx->sx_lock;
+
+#ifdef ADAPTIVE_SX
+		owner = (struct thread *)SX_HOLDER(x);
+		if (!(x & SX_SHARED_BIT) && (sx->sx_flags & SX_ADAPTIVESPIN) &&
+		    TD_IS_RUNNING(owner)) {
+			sleepq_release(&sx->lock_object);
+			cpu_spinwait();
+			continue;
+		}
+#endif
+
+		/*
+		 * The lock could be released while we spun.  In this case
+		 * loop back and retry.
+		 */
+		if (x & SX_SHARED_BIT) {
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
+
+		/*
+		 * We have to mantain consistent the SX_SHARED_WAITERS_BIT
+		 * flag, so try to set it (if not alredy set).  If we fail to
+		 * set it drop the sleepqueue lock and loop back.
+		 */
+		if (!(x & SX_SHARED_WAITERS_BIT) &&
+		    !atomic_cmpset_ptr(&sx->sx_lock, x, x |
+		    SX_SHARED_WAITERS_BIT)) {
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
+
+		/*
+		 * Since we have been unable to acquire the shared lock,
+		 * we have to sleep.
+		 */
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p blocking on sleepqueue",
+			    __func__, sx);
 
-	if (sx->sx_cnt == 1) {
-		sx->sx_cnt = -1;
-		sx->sx_xholder = curthread;
+		while (mtx_owned(&Giant)) {
+			gtcnt++;
+			mtx_unlock(&Giant);
+		}
+		sleepq_add(&sx->lock_object, NULL, sx->sx_desc, SLEEPQ_SX, 1);
+		sleepq_wait(&sx->lock_object);
 
-		LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, 1, file, line);
-		WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
-		    file, line);
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p resuming from sleepqueue",
+			    __func__, sx);
+	}
 
-		mtx_unlock(sx->sx_lock);
-		return (1);
-	} else {
-		LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, 0, file, line);
-		mtx_unlock(sx->sx_lock);
-		return (0);
+	while (gtcnt) {
+		mtx_lock(&Giant);
+		gtcnt--;
 	}
 }
 
+/*
+ * This function rappresents the so-called 'hard case' for sx_sunlock operation.
+ * All 'easy case' failures are redirected to this.  Note that ideally
+ * this might be a static function, but it needs to be accessible at least
+ * from sx.h.
+ */
 void
-_sx_downgrade(struct sx *sx, const char *file, int line)
+_sx_sunlock_hard(struct sx *sx, const char *file, int line)
 {
+	uintptr_t x;
+
+	for (;;) {
+		x = sx->sx_lock;
+
+		/*
+		 * We should never have sharers while at least one thread
+		 * holds a shared lock.
+		 */
+		KASSERT(!(x & SX_SHARED_WAITERS_BIT),
+		    ("%s: waiting sharers", __func__));
+
+		/*
+		 * See if there is more than one read lock held.  If so,
+		 * just drop one and return.
+		 */
+		if (SX_SHARERS(x) > 1) {
+			if (atomic_cmpset_ptr(&sx->sx_lock, x,
+			    x - SX_ONE_SHARER))
+				break;
+			continue;
+		}
 
-	_sx_assert(sx, SX_XLOCKED, file, line);
-	mtx_lock(sx->sx_lock);
-	MPASS(sx->sx_cnt == -1);
+		/*
+		 * If there aren't any waiters for an exclusive lock, then try
+		 * to drop it quickly.
+		 */
+		if (!(x & SX_EXCLUSIVE_WAITERS_BIT)) {
+			MPASS(x == SX_SHARERS_LOCK(1));
+			if (atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1),
+			    SX_UNHELD))
+				break;
+			continue;
+		}
 
-	WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
+		/*
+		 * At this point, there should just be one sharer with
+		 * exclusive waiters.
+		 */
+		MPASS(x == (SX_SHARERS_LOCK(1) | SX_EXCLUSIVE_WAITERS_BIT));
 
-	sx->sx_cnt = 1;
-	sx->sx_xholder = NULL;
-        if (sx->sx_shrd_wcnt > 0)
-                cv_broadcast(&sx->sx_shrd_cv);
+		sleepq_lock(&sx->lock_object);
 
-	LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
+		/*
+		 * Wake up semantic here is mantained quite simple:
+		 * Just wake up all the exclusive waiters.
+		 * Note that the state of the lock could be changed, so if it
+		 * fails loop back and retry.
+		 */
+		if (!atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
+		    SX_EXCLUSIVE_WAITERS_BIT, SX_UNHELD)) {
+			sleepq_release(&sx->lock_object);
+			continue;
+		}
+		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+			CTR2(KTR_LOCK, "%s: %p waking up all thread on"
+			    "exclusive queue", __func__, sx);
+		sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 0);
 
-	mtx_unlock(sx->sx_lock);
+		break;
+	}
 }
 
 #ifdef INVARIANT_SUPPORT
@@ -380,40 +743,61 @@
 		return;
 	switch (what) {
 	case SX_LOCKED:
-	case SX_LOCKED | LA_NOTRECURSED:
+	case SX_LOCKED | SX_NOTRECURSED:
+	case SX_LOCKED | SX_RECURSED:
 	case SX_SLOCKED:
 #ifdef WITNESS
 		witness_assert(&sx->lock_object, what, file, line);
 #else
-		mtx_lock(sx->sx_lock);
-		if (sx->sx_cnt <= 0 &&
-		    (what == SX_SLOCKED || sx->sx_xholder != curthread))
+		/*
+		 * If some other thread has a write lock or we have one
+		 * and are asserting a read lock, fail.  Also, if no one has
+		 * a lock at all, fail.
+		 */
+		if (sx->sx_lock == SX_UNHELD || (!(sx->sx_lock &
+		    SX_SHARED_BIT) && (what == SX_SLOCKED || sx_xholder(sx) !=
+		    curthread)))
 			panic("Lock %s not %slocked @ %s:%d\n",
 			    sx->lock_object.lo_name, (what == SX_SLOCKED) ?
 			    "share " : "", file, line);
-		mtx_unlock(sx->sx_lock);
+
+		if (!(sx->sx_lock & SX_SHARED_BIT)) {
+			if (sx_recursed(sx)) {
+				if (what & SX_NOTRECURSED)
+					panic("Lock %s not recursed @ %s:%d\n",
+					    sx->lock_object.lo_name, file,
+					    line);
+		 	} else if (what & SX_RECURSED)
+				panic("Lock %s recursed @ %s:%d\n",
+				    sx->lock_object.lo_name, file, line);
+		}
 #endif
 		break;
 	case SX_XLOCKED:
-		mtx_lock(sx->sx_lock);
-		if (sx->sx_xholder != curthread)
+	case SX_XLOCKED | SX_NOTRECURSED:
+	case SX_XLOCKED | SX_RECURSED:
+		if (sx_xholder(sx) != curthread)
 			panic("Lock %s not exclusively locked @ %s:%d\n",
 			    sx->lock_object.lo_name, file, line);
-		mtx_unlock(sx->sx_lock);
+		if (sx_recursed(sx)) {
+			if (what & SX_NOTRECURSED)
+				panic("Lock %s not recursed @ %s:%d\n",
+				    sx->lock_object.lo_name, file, line);
+		} else if (what & SX_RECURSED)
+			panic("Lock %s recursed @ %s:%d\n",
+			    sx->lock_object.lo_name, file, line);
 		break;
 	case SX_UNLOCKED:
 #ifdef WITNESS
 		witness_assert(&sx->lock_object, what, file, line);
 #else
 		/*
-		 * We are able to check only exclusive lock here,
-		 * we cannot assert that *this* thread owns slock.
+		 * If we hold a write lock fail.  We can't reliably check
+		 * to see if we hold a read lock or not.
 		 */
-		mtx_lock(sx->sx_lock);
-		if (sx->sx_xholder == curthread)
+		if (sx_xholder(sx) == curthread)
 			panic("Lock %s exclusively locked @ %s:%d\n",
 			    sx->lock_object.lo_name, file, line);
-		mtx_unlock(sx->sx_lock);
 #endif
 		break;
 	default:
@@ -424,32 +808,42 @@
 #endif	/* INVARIANT_SUPPORT */
 
 #ifdef DDB
-void
+static void
 db_show_sx(struct lock_object *lock)
 {
-	struct thread *td;
-	struct sx *sx;
+        struct thread *td;
+        struct sx *sx;
+
+        sx = (struct sx *)lock;
 
-	sx = (struct sx *)lock;
+        db_printf(" state: ");
+        if (sx->sx_lock == SX_UNHELD)
+                db_printf("UNLOCKED\n");
+        else if (sx->sx_lock & SX_SHARED_BIT)
+                db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
+        else {
+                td = sx_xholder(sx);
+                db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
+                    td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
+        }
 
-	db_printf(" state: ");
-	if (sx->sx_cnt < 0) {
-		td = sx->sx_xholder;
-		db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
-		    td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
-	} else if (sx->sx_cnt > 0)
-		db_printf("SLOCK: %d locks\n", sx->sx_cnt);
-	else
-		db_printf("UNLOCKED\n");
-	db_printf(" waiters: %d shared, %d exclusive\n", sx->sx_shrd_wcnt,
-	    sx->sx_excl_wcnt);
+	db_printf(" waiters: ");
+	switch(sx->sx_lock & (SX_SHARED_WAITERS_BIT |
+	    SX_EXCLUSIVE_WAITERS_BIT)) {
+		case SX_EXCLUSIVE_WAITERS_BIT:
+			db_printf("shared\n");
+		break;
+		case SX_SHARED_WAITERS_BIT:
+			db_printf("exclusive\n");
+		break;
+		case SX_SHARED_WAITERS_BIT | SX_EXCLUSIVE_WAITERS_BIT:
+			db_printf("exclusive and shared\n");
+		break;
+		default:
+			db_printf("none\n");
+	}
 }
 
-/*
- * Check to see if a thread that is blocked on a sleep queue is actually
- * blocked on an sx lock.  If so, output some details and return true.
- * If the lock has an exclusive owner, return that in *ownerp.
- */
 int
 sx_chain(struct thread *td, struct thread **ownerp)
 {
@@ -457,43 +851,34 @@
 	struct cv *cv;
 
 	/*
-	 * First, see if it looks like td is blocked on a condition
-	 * variable.
+	 * First, see if it looks like td is blocked on a condition variable.
 	 */
 	cv = td->td_wchan;
 	if (cv->cv_description != td->td_wmesg)
 		return (0);
 
 	/*
-	 * Ok, see if it looks like td is blocked on the exclusive
-	 * condition variable.
+	 * Ok, see if it is like td isn't blocked on an sx.
 	 */
-	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_excl_cv));
-	if (LOCK_CLASS(&sx->lock_object) == &lock_class_sx &&
-	    sx->sx_excl_wcnt > 0)
-		goto ok;
+	sx = td->td_wchan;
+	if (sx->sx_desc != td->td_wmesg ||
+	    LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
+	    (!(sx->sx_lock & SX_SHARED_WAITERS_BIT) && !(sx->sx_lock &
+	    SX_EXCLUSIVE_WAITERS_BIT)))
+		return (0);
 
-	/*
-	 * Second, see if it looks like td is blocked on the shared
-	 * condition variable.
-	 */
-	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_shrd_cv));
-	if (LOCK_CLASS(&sx->lock_object) == &lock_class_sx &&
-	    sx->sx_shrd_wcnt > 0)
-		goto ok;
-
-	/* Doesn't seem to be an sx lock. */
-	return (0);
-
-ok:
-	/* We think we have an sx lock, so output some details. */
-	db_printf("blocked on sx \"%s\" ", td->td_wmesg);
-	if (sx->sx_cnt >= 0) {
-		db_printf("SLOCK (count %d)\n", sx->sx_cnt);
+	db_printf("blocked on sx lock");
+	if (sx->sx_lock == SX_UNHELD) {
+		db_printf("UNHELD\n");
+		*ownerp = NULL;
+	} else if (sx->sx_lock & SX_SHARED_BIT) {
+		db_printf("SHARED (count %ju)\n",
+		    (uintmax_t)SX_SHARERS(sx->sx_lock));
 		*ownerp = NULL;
 	} else {
-		db_printf("XLOCK\n");
-		*ownerp = sx->sx_xholder;
+		db_printf("EXCL (holder: 0x%jx)\n",
+		    (uintmax_t)SX_HOLDER(sx->sx_lock));
+		*ownerp = (struct thread *)SX_HOLDER(sx->sx_lock);
 	}
 	return (1);
 }
--- //depot/vendor/freebsd/src/sys/kern/kern_thread.c	2007/03/09 04:07:02
+++ //depot/user/attilio/attilio_smpng/kern/kern_thread.c	2007/03/10 19:39:06
@@ -247,7 +247,7 @@
 
 	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
 	    thread_ctor, thread_dtor, thread_init, thread_fini,
-	    UMA_ALIGN_CACHE, 0);
+	    16 - 1, 0);
 #ifdef KSE
 	kseinit();	/* set up kse specific stuff  e.g. upcall zone*/
 #endif
--- //depot/vendor/freebsd/src/sys/kern/vfs_mount.c	2007/03/18 02:41:54
+++ //depot/user/attilio/attilio_smpng/kern/vfs_mount.c	2007/03/18 15:17:54
@@ -43,6 +43,7 @@
 #include <sys/jail.h>
 #include <sys/kernel.h>
 #include <sys/libkern.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
 #include <sys/mutex.h>
--- //depot/vendor/freebsd/src/sys/kern/vfs_syscalls.c	2007/03/21 19:36:52
+++ //depot/user/attilio/attilio_smpng/kern/vfs_syscalls.c	2007/03/22 15:33:45
@@ -45,6 +45,7 @@
 #include <sys/bio.h>
 #include <sys/buf.h>
 #include <sys/sysent.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
 #include <sys/mutex.h>
--- //depot/vendor/freebsd/src/sys/netinet6/in6_src.c	2006/12/12 12:23:39
+++ //depot/user/attilio/attilio_smpng/netinet6/in6_src.c	2006/12/16 21:33:24
@@ -66,6 +66,7 @@
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/priv.h>
--- //depot/vendor/freebsd/src/sys/security/mac_biba/mac_biba.c	2007/02/23 14:41:13
+++ //depot/user/attilio/attilio_smpng/security/mac_biba/mac_biba.c	2007/03/04 00:12:48
@@ -44,6 +44,7 @@
 #include <sys/extattr.h>
 #include <sys/kernel.h>
 #include <sys/ksem.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
 #include <sys/mount.h>
--- //depot/vendor/freebsd/src/sys/security/mac_lomac/mac_lomac.c	2007/02/06 14:22:42
+++ //depot/user/attilio/attilio_smpng/security/mac_lomac/mac_lomac.c	2007/02/09 22:24:44
@@ -45,6 +45,7 @@
 #include <sys/conf.h>
 #include <sys/extattr.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
 #include <sys/mount.h>
--- //depot/vendor/freebsd/src/sys/security/mac_mls/mac_mls.c	2007/02/06 14:22:42
+++ //depot/user/attilio/attilio_smpng/security/mac_mls/mac_mls.c	2007/02/09 22:24:44
@@ -46,6 +46,7 @@
 #include <sys/extattr.h>
 #include <sys/kernel.h>
 #include <sys/ksem.h>
+#include <sys/lock.h>
 #include <sys/mman.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
--- //depot/vendor/freebsd/src/sys/security/mac_none/mac_none.c	2007/02/23 11:22:44
+++ //depot/user/attilio/attilio_smpng/security/mac_none/mac_none.c	2007/03/04 00:12:48
@@ -44,6 +44,7 @@
 
 #include <sys/param.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/module.h>
 
 #include <security/mac/mac_policy.h>
--- //depot/vendor/freebsd/src/sys/security/mac_partition/mac_partition.c	2007/02/23 14:41:13
+++ //depot/user/attilio/attilio_smpng/security/mac_partition/mac_partition.c	2007/03/04 00:12:48
@@ -41,6 +41,7 @@
 
 #include <sys/param.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/module.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
--- //depot/vendor/freebsd/src/sys/security/mac_stub/mac_stub.c	2007/02/06 11:02:49
+++ //depot/user/attilio/attilio_smpng/security/mac_stub/mac_stub.c	2007/02/09 22:24:44
@@ -52,6 +52,7 @@
 #include <sys/extattr.h>
 #include <sys/kernel.h>
 #include <sys/ksem.h>
+#include <sys/lock.h>
 #include <sys/mount.h>
 #include <sys/proc.h>
 #include <sys/systm.h>
--- //depot/vendor/freebsd/src/sys/security/mac_test/mac_test.c	2007/02/23 14:41:13
+++ //depot/user/attilio/attilio_smpng/security/mac_test/mac_test.c	2007/03/04 00:12:48
@@ -44,6 +44,7 @@
 #include <sys/kdb.h>
 #include <sys/kernel.h>
 #include <sys/ksem.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/mount.h>
--- //depot/vendor/freebsd/src/sys/sys/lock.h	2007/03/21 22:22:41
+++ //depot/user/attilio/attilio_smpng/sys/lock.h	2007/03/22 15:33:45
@@ -326,11 +326,12 @@
  * wherever they please without having to actually grab a lock to do so.
  */
 #define	witness_check(l)						\
-	WITNESS_CHECKORDER(&(l)->lock_object, LOP_EXCLUSIVE, LOCK_FILE,	\
+	WITNESS_CHECKORDER(&(l)->lock_object, LOP_NEWORDER |		\
+	    LOP_EXCLUSIVE, LOCK_FILE, LOCK_LINE)
+
+#define	witness_check_shared(l)						\
+	WITNESS_CHECKORDER(&(l)->lock_object, LOP_NEWORDER, LOCK_FILE,	\
 	    LOCK_LINE)
 
-#define	witness_check_shared(l)						\
-	WITNESS_CHECKORDER(&(l)->lock_object, 0, LOCK_FILE, LOCK_LINE)
-	
 #endif	/* _KERNEL */
 #endif	/* _SYS_LOCK_H_ */
--- //depot/vendor/freebsd/src/sys/sys/sleepqueue.h	2007/03/09 22:41:06
+++ //depot/user/attilio/attilio_smpng/sys/sleepqueue.h	2007/03/10 19:39:06
@@ -33,8 +33,8 @@
 #define _SYS_SLEEPQUEUE_H_
 
 /*
- * Sleep queue interface.  Sleep/wakeup and condition variables use a sleep
- * queue for the queue of threads blocked on a sleep channel.
+ * Sleep queue interface.  Sleep/wakeup, condition variables and sx locks
+ * use a sleep queue for the queue of threads blocked on a sleep channel.
  *
  * A thread calls sleepq_lock() to lock the sleep queue chain associated
  * with a given wait channel.  A thread can then call call sleepq_add() to
@@ -85,6 +85,7 @@
 #define	SLEEPQ_SLEEP		0x00		/* Used by sleep/wakeup. */
 #define	SLEEPQ_CONDVAR		0x01		/* Used for a cv. */
 #define	SLEEPQ_PAUSE		0x02		/* Used by pause. */
+#define	SLEEPQ_SX		0x04		/* Used for a sx lock. */
 #define	SLEEPQ_INTERRUPTIBLE	0x100		/* Sleep is interruptible. */
 
 void	init_sleepqueues(void);
--- //depot/vendor/freebsd/src/sys/sys/sx.h	2007/03/21 21:22:46
+++ //depot/user/attilio/attilio_smpng/sys/sx.h	2007/03/22 16:52:36
@@ -1,5 +1,7 @@
 /*-
- * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>.  All rights reserved.
+ * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
+ * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -23,31 +25,113 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  * DAMAGE.
- *
- * $FreeBSD: src/sys/sys/sx.h,v 1.28 2007/03/21 21:20:51 jhb Exp $
  */
 
 #ifndef	_SYS_SX_H_
 #define	_SYS_SX_H_
 
-#include <sys/queue.h>
 #include <sys/_lock.h>
-#include <sys/condvar.h>	/* XXX */
+#include <sys/_sx.h>
+
+#ifdef	_KERNEL
+#include <machine/atomic.h>
+#endif
+
+/*
+ * The sx_lock member can be considered as a double-side bitmap.
+ * This means that when a sx lock is held as shared lock (one side), lower bits
+ * are reserved for lock manage and the other are used as a count for
+ * sharers.
+ * Otherwise, when a sx lock is held as an exclusive lock (the other side),
+ * lower bits are reserved for lock manage and the other are used in order to
+ * track the (singular) holder.
+ *
+ * When the lock is not locked by any thread, it is encoded as a shared lock
+ * with zero waiters.
+ *
+ * A note about memory barriers.  Exclusive locks need to use the same memory
+ * barriers as mutexes: _acq when acquiring an exclusive lock and _rel when
+ * releasing an exclusive lock.  On the other side, shared lock needs to use
+ * an _acq barrier when acquiring the lock but, since they don't update any
+ * locked data, no memory barrier is needed when releasing a shared lock.
+ */
+#define	SX_SHARED_BIT			0x01
+#define	SX_SHARED_WAITERS_BIT		0x02
+#define	SX_EXCLUSIVE_WAITERS_BIT	0x04
+#define	SX_RECURSED_BIT			0x08
+#define	SX_FLAGMASK							\
+	(SX_SHARED_BIT | SX_SHARED_WAITERS_BIT |			\
+	SX_EXCLUSIVE_WAITERS_BIT | SX_RECURSED_BIT)
 
-struct sx {
-	struct lock_object lock_object;	/* Common lock properties. */
-	struct mtx	*sx_lock;	/* General protection lock. */
-	int		sx_cnt;		/* -1: xlock, > 0: slock count. */
-	struct cv	sx_shrd_cv;	/* slock waiters. */
-	int		sx_shrd_wcnt;	/* Number of slock waiters. */
-	struct cv	sx_excl_cv;	/* xlock waiters. */
-	int		sx_excl_wcnt;	/* Number of xlock waiters. */
-	struct thread	*sx_xholder;	/* Thread presently holding xlock. */
-};
+#define	SX_HOLDER(x)			((x) & ~SX_FLAGMASK)
+#define	SX_SHARERS_SHIFT		4
+#define	SX_SHARERS(x)			(SX_HOLDER(x) >> SX_SHARERS_SHIFT)
+#define	SX_SHARERS_LOCK(x)		((x) << SX_SHARERS_SHIFT |	\
+					SX_SHARED_BIT)
+#define	SX_ONE_SHARER			(1 << SX_SHARERS_SHIFT)
+#define	SX_UNHELD			SX_SHARERS_LOCK(0)
 
 #ifdef _KERNEL
+
+/*
+ * Full lock operations that are suitable to be inlined in non-debug kernels.
+ * If the lock can't be acquired or released trivially then the work is
+ * deferred to 'tougher' functions.
+ */
+
+/* Acquire an exclusive lock. */
+#define	__sx_xlock(sx, tid, file, line) do {				\
+		uintptr_t _tid = (uintptr_t)(tid);			\
+									\
+		if (!atomic_cmpset_acq_ptr(&(sx)->sx_lock, SX_UNHELD,	\
+		    _tid))						\
+			_sx_xlock_hard((sx), _tid, (file), (line));	\
+} while (0)
+
+/* Release an exclusive lock. */
+#define	__sx_xunlock(sx, tid, file, line) do {				\
+		uintptr_t _tid = (uintptr_t)(tid);			\
+									\
+		if (!atomic_cmpset_rel_ptr(&(sx)->sx_lock, _tid,	\
+		    SX_UNHELD))						\
+			_sx_xunlock_hard((sx), _tid, (file), (line));	\
+} while (0)
+
+/* Acquire a shared lock. */
+#define	__sx_slock(sx, file, line) do {					\
+		uintptr_t x = (sx)->sx_lock;				\
+									\
+		if (x & SX_SHARED_BIT) {				\
+			if (!atomic_cmpset_acq_ptr(&(sx)->sx_lock, x,	\
+			    x + SX_ONE_SHARER))				\
+				_sx_slock_hard((sx), (file), (line));	\
+		} else							\
+			_sx_slock_hard((sx), (file), (line));		\
+} while (0)
+
+/* Release a shared lock. */
+#define	__sx_sunlock(sx, file, line) do {				\
+		uintptr_t x = (sx)->sx_lock;				\
+									\
+		if (SX_SHARERS(x) > 1) {				\
+			if (!atomic_cmpset_ptr(&(sx)->sx_lock, x,	\
+			    x - SX_ONE_SHARER))				\
+				_sx_sunlock_hard((sx), (file), (line));	\
+		} else if (!(x & SX_EXCLUSIVE_WAITERS_BIT)) {		\
+			if(!atomic_cmpset_ptr(&(sx)->sx_lock,		\
+			    SX_SHARERS_LOCK(1), SX_UNHELD))		\
+				_sx_sunlock_hard((sx), (file), (line));	\
+		} else							\
+			_sx_sunlock_hard((sx), (file), (line));		\
+} while (0)
+
+/*
+ * Function prototipes.  Routines that start with an underscore are not part
+ * of the public interface and might be wrappered with a macro.
+ */
 void	sx_sysinit(void *arg);
 void	sx_init(struct sx *sx, const char *description);
+void	sx_init_flags(struct sx *sx, const char *description, int opts);
 void	sx_destroy(struct sx *sx);
 void	_sx_slock(struct sx *sx, const char *file, int line);
 void	_sx_xlock(struct sx *sx, const char *file, int line);
@@ -57,6 +141,12 @@
 void	_sx_xunlock(struct sx *sx, const char *file, int line);
 int	_sx_try_upgrade(struct sx *sx, const char *file, int line);
 void	_sx_downgrade(struct sx *sx, const char *file, int line);
+void	_sx_xlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
+	    line);
+void	_sx_slock_hard(struct sx *sx, const char *file, int line);
+void	_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
+	    line);
+void	_sx_sunlock_hard(struct sx *sx, const char *file, int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
 void	_sx_assert(struct sx *sx, int what, const char *file, int line);
 #endif
@@ -79,29 +169,63 @@
 	SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE,	\
 	    sx_destroy, (sxa))
 
-#define	sx_xlocked(sx)		((sx)->sx_cnt < 0 && (sx)->sx_xholder == curthread)
+/*
+ * Public interface for lock operations.
+ */
+#ifndef LOCK_DEBUG
+#error	"LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>"
+#endif
+#if	(LOCK_DEBUG > 0) || defined(SXLOCK_NOINLINE)
+#define	sx_xlock(sx)		_sx_xlock((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_xunlock(sx)		_sx_xunlock((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_slock(sx)		_sx_slock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_xlock(sx)		_sx_xlock((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_sunlock(sx)		_sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
+#else
+#define	sx_xlock(sx)							\
+	__sx_xlock((sx), curthread, LOCK_FILE, LOCK_LINE)
+#define	sx_xunlock(sx)							\
+	__sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE)
+#define	sx_slock(sx)		__sx_slock((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_sunlock(sx)		__sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
+#endif	/* LOCK_DEBUG > 0 || SXLOCK_NOINLINE */
 #define	sx_try_slock(sx)	_sx_try_slock((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_try_xlock(sx)	_sx_try_xlock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_sunlock(sx)		_sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_xunlock(sx)		_sx_xunlock((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_try_upgrade(sx)	_sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE)
 #define	sx_downgrade(sx)	_sx_downgrade((sx), LOCK_FILE, LOCK_LINE)
+
+#define	sx_xlocked(sx)							\
+	((sx)->sx_lock & SX_SHARED_BIT ? 0 :				\
+	SX_HOLDER((sx)->sx_lock) == (uintptr_t)curthread)
+
 #define	sx_unlock(sx) do {						\
 	if (sx_xlocked(sx))						\
 		sx_xunlock(sx);						\
 	else								\
 		sx_sunlock(sx);						\
 } while (0)
+
 #define	sx_sleep(chan, sx, pri, wmesg, timo)				\
 	_sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo))
 
+/*
+ * Options passed to sx_init_flags().
+ */
+#define	SX_DUPOK		0x01
+#define	SX_NOPROFILE		0x02
+#define	SX_NOWITNESS		0x04
+#define	SX_QUIET		0x08
+#define	SX_ADAPTIVESPIN		0x10
+
+/*
+ * XXX: These options should be renamed as SA_*
+ */
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
 #define	SX_LOCKED		LA_LOCKED
 #define	SX_SLOCKED		LA_SLOCKED
 #define	SX_XLOCKED		LA_XLOCKED
 #define	SX_UNLOCKED		LA_UNLOCKED
+#define	SX_RECURSED		LA_RECURSED
+#define	SX_NOTRECURSED		LA_NOTRECURSED
 #endif
 
 #ifdef INVARIANTS
