--- /usr/src/sys/sys/_rwlock.h	Tue Jan 31 19:08:39 2006
+++ sys/sys/_rwlock.h	Mon Aug  7 06:57:31 2006
@@ -38,6 +38,12 @@
 struct rwlock {
 	struct lock_object	rw_object;
 	volatile uintptr_t	rw_lock;
+	volatile uintptr_t	rw_oor;
 };
+
+/*
+ * Owner of records handling macro.
+ */
+#define	OOR_UNOWNED		((uintptr_t)NULL)
 
 #endif /* !_SYS__RWLOCK_H_ */
--- /usr/src/sys/sys/rwlock.h	Fri Jun 23 21:36:50 2006
+++ sys/sys/rwlock.h	Mon Aug  7 05:24:49 2006
@@ -64,16 +64,28 @@
 #define	RW_LOCK_READ		0x01
 #define	RW_LOCK_READ_WAITERS	0x02
 #define	RW_LOCK_WRITE_WAITERS	0x04
-#define	RW_LOCK_FLAGMASK						\
+#define	RW_LOCK_OWNED		0x08
+#define	RW_LOCK_EXEMPTED	0x10
+/*
+ * read and write locks need different flagmask in order to not hurt thread
+ * memory boundaries.
+ */
+#define	RW_LOCK_WFLAGMASK						\
 	(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)
+#define	RW_LOCK_RFLAGMASK						\
+	(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS |	\
+	    RW_LOCK_OWNED | RW_LOCK_EXEMPTED)
 
-#define	RW_OWNER(x)		((x) & ~RW_LOCK_FLAGMASK)
-#define	RW_READERS_SHIFT	3
-#define	RW_READERS(x)		(RW_OWNER((x)) >> RW_READERS_SHIFT)
+#define	RW_WOWNER(x)		((x) & ~RW_LOCK_WFLAGMASK)
+#define	RW_RCOUNT(x)		((x) & ~RW_LOCK_RFLAGMASK)
+#define	RW_READERS_SHIFT	5
+#define	RW_READERS(x)		(RW_RCOUNT((x)) >> RW_READERS_SHIFT)
 #define	RW_READERS_LOCK(x)	((x) << RW_READERS_SHIFT | RW_LOCK_READ)
 #define	RW_ONE_READER		(1 << RW_READERS_SHIFT)
+#define	RW_READER_FREE		(RW_LOCK_OWNED | RW_LOCK_EXEMPTED)
 
 #define	RW_UNLOCKED		RW_READERS_LOCK(0)
+#define	RW_UNOWNED		OOR_UNOWNED
 
 #ifdef _KERNEL
 
--- /usr/src/sys/kern/kern_rwlock.c	Wed Apr 19 23:06:52 2006
+++ sys/kern/kern_rwlock.c	Tue Aug  8 04:36:47 2006
@@ -62,19 +62,23 @@
 };
 
 /*
- * Return a pointer to the owning thread if the lock is write-locked or
- * NULL if the lock is unlocked or read-locked.
+ * Return a pointer to the owner of records if set or NULL otherwise.
+ */
+#define	rw_rowner(rw)							\
+	((struct thread *)(rw)->rw_oor)
+
+/*
+ * Return a pointer to the owning thread.
  */
 #define	rw_wowner(rw)							\
-	((rw)->rw_lock & RW_LOCK_READ ? NULL :				\
-	    (struct thread *)RW_OWNER((rw)->rw_lock))
+	((struct thread *)RW_WOWNER((rw)->rw_lock))
 
 /*
  * Return a pointer to the owning thread for this lock who should receive
- * any priority lent by threads that block on this lock.  Currently this
- * is identical to rw_wowner().
+ * any priority lent by threads that block on this lock.
  */
-#define	rw_owner(rw)		rw_wowner(rw)
+#define	rw_owner(rw)							\
+	((rw)->rw_lock &  RW_LOCK_READ ? rw_rowner(rw) : rw_wowner(rw))
 
 #ifndef INVARIANTS
 #define	_rw_assert(rw, what, file, line)
@@ -85,6 +89,7 @@
 {
 
 	rw->rw_lock = RW_UNLOCKED;
+	rw->rw_oor = RW_UNOWNED;
 
 	lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS |
 	    LO_RECURSABLE | LO_UPGRADABLE);
@@ -95,6 +100,7 @@
 {
 
 	KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked"));
+	KASSERT(rw->rw_oor == RW_UNOWNED, ("rw lock OoR is set on exit"));
 	lock_destroy(&rw->rw_object);
 }
 
@@ -138,7 +144,7 @@
 #ifdef SMP
 	volatile struct thread *owner;
 #endif
-	uintptr_t x;
+	uintptr_t tid, x;
 
 	KASSERT(rw_wowner(rw) != curthread,
 	    ("%s (%s): wlock already held @ %s:%d", __func__,
@@ -167,6 +173,7 @@
 		 * completely unlocked rwlock since such a lock is encoded
 		 * as a read lock with no waiters.
 		 */
+		tid = (uintptr_t)curthread;
 		x = rw->rw_lock;
 		if (x & RW_LOCK_READ) {
 
@@ -176,14 +183,54 @@
 			 * and in that case RW_LOCK_READ should be clear.
 			 */
 			MPASS((x & RW_LOCK_READ_WAITERS) == 0);
-			if (atomic_cmpset_acq_ptr(&rw->rw_lock, x,
-			    x + RW_ONE_READER)) {
-				if (LOCK_LOG_TEST(&rw->rw_object, 0))
-					CTR4(KTR_LOCK,
-					    "%s: %p succeed %p -> %p", __func__,
-					    rw, (void *)x,
-					    (void *)(x + RW_ONE_READER));
-				break;
+
+			/*
+			 * If the owner of records is not alredy set, go in
+			 * the hard case.
+			 */
+			if (x & RW_LOCK_OWNED) {
+				if (atomic_cmpset_acq_ptr(&rw->rw_lock, x,
+				    x + RW_ONE_READER)) {
+					if (LOCK_LOG_TEST(&rw->rw_object, 0))
+						CTR4(KTR_LOCK,
+						    "%s: %p succeed %p -> %p",
+						    __func__, rw, (void *)x,
+					    	    (void*)(x + RW_ONE_READER));
+					break;
+				}
+			} else {
+				/*
+				 * Set the owner of records here.
+				 * This is the first rlock(), so all the
+				 * "extra" bits might be disabled as default.
+				 * Note that all RW_LOCK_OWNED and
+				 * RW_LOCK_EXEMPTED bits manipulations here
+				 * happen with turnstile locked beacause we
+				 * need to sync with turnstile_wait().
+				 */
+				turnstile_lock(&rw->rw_object);
+				x = rw->rw_lock;
+				MPASS(!(x & RW_LOCK_EXEMPTED));
+				MPASS(!(x & RW_LOCK_WRITE_WAITERS));
+
+				/*
+				 * The OoR could be set while we spun, so
+				 * drop turnstile spin-lock and continue.
+				 */
+				if (!(x & RW_LOCK_OWNED)) {
+					/*
+					 * If the owner of records is
+					 * successfully acquired than set the
+					 * RW_LOCK_OWNED bit in order to handle
+					 * the easy case.
+					 */
+					if (atomic_cmpset_acq_ptr(&rw->rw_lock,
+					    x, x | RW_LOCK_OWNED))
+						atomic_set_acq_ptr(&rw->rw_oor,
+						    tid);
+				}
+
+				turnstile_release(&rw->rw_object);
 			}
 			cpu_spinwait();
 			continue;
@@ -232,13 +279,13 @@
 		 * the owner stops running or the state of the lock
 		 * changes.
 		 */
-		owner = (struct thread *)RW_OWNER(x);
+		owner = (struct thread *)RW_WOWNER(x);
 		if (TD_IS_RUNNING(owner)) {
 			turnstile_release(&rw->rw_object);
 			if (LOCK_LOG_TEST(&rw->rw_object, 0))
 				CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
 				    __func__, rw, owner);
-			while ((struct thread*)RW_OWNER(rw->rw_lock)== owner &&
+			while ((struct thread*)RW_WOWNER(rw->rw_lock)== owner &&
 			    TD_IS_RUNNING(owner))
 				cpu_spinwait();
 			continue;
@@ -258,12 +305,6 @@
 			    __func__, rw);
 	}
 
-	/*
-	 * TODO: acquire "owner of record" here.  Here be turnstile dragons
-	 * however.  turnstiles don't like owners changing between calls to
-	 * turnstile_wait() currently.
-	 */
-
 	LOCK_LOG_LOCK("RLOCK", &rw->rw_object, 0, 0, file, line);
 	WITNESS_LOCK(&rw->rw_object, 0, file, line);
 }
@@ -272,20 +313,49 @@
 _rw_runlock(struct rwlock *rw, const char *file, int line)
 {
 	struct turnstile *ts;
-	uintptr_t x;
+	uintptr_t tid, x;
 
 	_rw_assert(rw, RA_RLOCKED, file, line);
 	WITNESS_UNLOCK(&rw->rw_object, 0, file, line);
 	LOCK_LOG_LOCK("RUNLOCK", &rw->rw_object, 0, 0, file, line);
 
-	/* TODO: drop "owner of record" here. */
-
 	for (;;) {
 		/*
 		 * See if there is more than one read lock held.  If so,
 		 * just drop one and return.
 		 */
+		tid = (uintptr_t)curthread;
 		x = rw->rw_lock;
+		MPASS(x & RW_LOCK_OWNED);
+		MPASS(!(x & RW_LOCK_READ_WAITERS));
+
+		/*
+		 * Here there is not a race beacause it is assumed that
+		 * RW_LOCK_OWNED is set and rw->rw_oor has just two states:
+		 * { RW_UNOWNED, curthread }.  Here we need to syncronize with
+		 * RW_LOCK_WRITE_WAITERS bit changes so lock the turnstile.
+		 */
+		if (!(x & RW_LOCK_EXEMPTED) && rw->rw_oor == tid) {
+			turnstile_lock(&rw->rw_object);
+			x = rw->rw_lock;
+
+			if (!(x & RW_LOCK_EXEMPTED) &&
+			    atomic_cmpset_acq_ptr(&rw->rw_lock, x,
+			    x | RW_LOCK_EXEMPTED)) {
+				atomic_store_rel_ptr(&rw->rw_oor, RW_UNOWNED);
+				if (x & RW_LOCK_WRITE_WAITERS) {
+					ts = turnstile_lookup(&rw->rw_object);
+					turnstile_disown(ts);
+					cpu_spinwait();
+					continue;
+				}
+			}
+
+			turnstile_release(&rw->rw_object);
+			cpu_spinwait();
+			continue;
+		}
+
 		if (RW_READERS(x) > 1) {
 			if (atomic_cmpset_ptr(&rw->rw_lock, x,
 			    x - RW_ONE_READER)) {
@@ -319,9 +389,7 @@
 			 * have just acquired a read lock, so go back up
 			 * to the multiple read locks case.
 			 */
-			MPASS(x == RW_READERS_LOCK(1));
-			if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1),
-			    RW_UNLOCKED)) {
+			if (atomic_cmpset_ptr(&rw->rw_lock, x, RW_UNLOCKED)) {
 				if (LOCK_LOG_TEST(&rw->rw_object, 0))
 					CTR2(KTR_LOCK, "%s: %p last succeeded",
 					    __func__, rw);
@@ -331,16 +399,11 @@
 		}
 
 		/*
-		 * There should just be one reader with one or more
-		 * writers waiting.
-		 */
-		MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS));
-
-		/*
 		 * Ok, we know we have a waiting writer and we think we
 		 * are the last reader, so grab the turnstile lock.
 		 */
 		turnstile_lock(&rw->rw_object);
+		x = rw->rw_lock;
 
 		/*
 		 * Try to drop our lock leaving the lock in a unlocked
@@ -358,8 +421,8 @@
 		 * acquired a read lock, so drop the turnstile lock and
 		 * restart.
 		 */
-		if (!atomic_cmpset_ptr(&rw->rw_lock,
-		    RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) {
+		MPASS(!(x & RW_LOCK_READ_WAITERS));
+		if (!atomic_cmpset_ptr(&rw->rw_lock, x, RW_UNLOCKED)) {
 			turnstile_release(&rw->rw_object);
 			continue;
 		}
@@ -459,14 +522,16 @@
 		 * If the lock is write locked and the owner is
 		 * running on another CPU, spin until the owner stops
 		 * running or the state of the lock changes.
+		 * XXX: We might behave in this way even if the owner of
+		 * records is present?
 		 */
-		owner = (struct thread *)RW_OWNER(v);
+		owner = (struct thread *)RW_WOWNER(v);
 		if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
 			turnstile_release(&rw->rw_object);
 			if (LOCK_LOG_TEST(&rw->rw_object, 0))
 				CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
 				    __func__, rw, owner);
-			while ((struct thread*)RW_OWNER(rw->rw_lock)== owner &&
+			while ((struct thread*)RW_WOWNER(rw->rw_lock)== owner &&
 			    TD_IS_RUNNING(owner))
 				cpu_spinwait();
 			continue;
@@ -599,45 +664,35 @@
 	_rw_assert(rw, RA_RLOCKED, file, line);
 
 	/*
-	 * Attempt to switch from one reader to a writer.  If there
-	 * are any write waiters, then we will have to lock the
-	 * turnstile first to prevent races with another writer
-	 * calling turnstile_wait() before we have claimed this
-	 * turnstile.  So, do the simple case of no waiters first.
-	 */
-	tid = (uintptr_t)curthread;
-	if (!(rw->rw_lock & RW_LOCK_WRITE_WAITERS)) {
-		success = atomic_cmpset_acq_ptr(&rw->rw_lock,
-		    RW_READERS_LOCK(1), tid);
-		goto out;
-	}
-
-	/*
-	 * Ok, we think we have write waiters, so lock the
-	 * turnstile.
+	 * Attempt to switch from one reader to a writer. We have to hold
+	 * the turnstile spin-lock since we have to manipulate owner of records.
 	 */
 	turnstile_lock(&rw->rw_object);
+	tid = (uintptr_t)curthread;
+	v = rw->rw_lock & RW_LOCK_RFLAGMASK;
 
 	/*
-	 * Try to switch from one reader to a writer again.  This time
-	 * we honor the current state of the RW_LOCK_WRITE_WAITERS
-	 * flag.  If we obtain the lock with the flag set, then claim
-	 * ownership of the turnstile.  In the SMP case it is possible
-	 * for there to not be an associated turnstile even though there
-	 * are waiters if all of the waiters are spinning.
-	 */
-	v = rw->rw_lock & RW_LOCK_WRITE_WAITERS;
-	success = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
-	    tid | v);
+	 * Try to switch from one reader to a writer. The OoR has only two
+	 * available values: { RW_UNOWNED, curthread }, so if we can't set
+	 * clean rw_oor with a memory barrier the owner of records is not set
+	 * and we have to claim turnstile ownership, otherwise we just leave
+	 * the owner untouched (we have to consider the RW_LOCK_WRITE_WAITERS
+	 * flag too).
+	 */
+	success = atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
+	    tid | (v & (RW_LOCK_WFLAGMASK & ~RW_LOCK_READ)));
+	v &= RW_LOCK_WRITE_WAITERS;
 #ifdef SMP
-	if (success && v && turnstile_lookup(&rw->rw_object) != NULL)
+	if (success && !atomic_cmpset_rel_ptr(&rw->rw_oor, tid, RW_UNOWNED) &&
+	    v && turnstile_lookup(&rw->rw_object) != NULL)
 #else
-	if (success && v)
+	if (success && !atomic_cmpset_rel_ptr(&rw->rw_oor, tid, RW_UNOWNED) &&
+	    v))
 #endif
 		turnstile_claim(&rw->rw_object);
 	else
 		turnstile_release(&rw->rw_object);
-out:
+
 	LOCK_LOG_TRY("WUPGRADE", &rw->rw_object, 0, success, file, line);
 	if (success)
 		WITNESS_UPGRADE(&rw->rw_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
@@ -659,27 +714,18 @@
 	WITNESS_DOWNGRADE(&rw->rw_object, 0, file, line);
 
 	/*
-	 * Convert from a writer to a single reader.  First we handle
-	 * the easy case with no waiters.  If there are any waiters, we
-	 * lock the turnstile, "disown" the lock, and awaken any read
-	 * waiters.
-	 */
-	tid = (uintptr_t)curthread;
-	if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
-		goto out;
-
-	/*
-	 * Ok, we think we have waiters, so lock the turnstile so we can
-	 * read the waiter flags without any races.
+	 * Convert from a writer to a single reader. We have to hold the
+	 * turnstile spin-lock in order to set the owner of records. If
+	 * there are any waiters we awaken any read waiters.
 	 */
 	turnstile_lock(&rw->rw_object);
+	tid = (uintptr_t)curthread;
 	v = rw->rw_lock;
-	MPASS(v & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS));
 
 	/*
 	 * Downgrade from a write lock while preserving
-	 * RW_LOCK_WRITE_WAITERS and give up ownership of the
-	 * turnstile.  If there are any read waiters, wake them up.
+	 * RW_LOCK_WRITE_WAITERS and set the owner of records here.
+	 * If there are any read waiters, wake them up.
 	 *
 	 * For SMP, we have to allow for the fact that all of the
 	 * read waiters might be spinning.  In that case, act as if
@@ -702,17 +748,17 @@
 #endif
 	if (v & RW_LOCK_READ_WAITERS)
 		turnstile_broadcast(ts, TS_SHARED_QUEUE);
-	atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) |
+	/*
+	 * Set the owner of records here.
+	 */
+	atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | RW_LOCK_OWNED |
 	    (v & RW_LOCK_WRITE_WAITERS));
+	atomic_store_rel_ptr(&rw->rw_oor, tid);
 	if (v & RW_LOCK_READ_WAITERS)
 		turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
-#ifdef SMP
-	else if (ts == NULL)
-		turnstile_release(&rw->rw_object);
-#endif
 	else
-		turnstile_disown(ts);
-out:
+		turnstile_release(&rw->rw_object);
+
 	LOCK_LOG_LOCK("WDOWNGRADE", &rw->rw_object, 0, 0, file, line);
 }
 
