--- //depot/vendor/freebsd/src/sys/amd64/amd64/intr_machdep.c	2007/05/31 19:33:13
+++ //depot/user/attilio/attilio_schedlock/amd64/amd64/intr_machdep.c	2007/06/04 10:32:52
@@ -250,7 +250,7 @@
 	 * processed too.
 	 */
 	(*isrc->is_count)++;
-	PCPU_LAZY_INC(cnt.v_intr);
+	PCPU_INC(cnt.v_intr);
 
 	ie = isrc->is_event;
 
@@ -321,7 +321,7 @@
 	 * processed too.
 	 */
 	(*isrc->is_count)++;
-	PCPU_LAZY_INC(cnt.v_intr);
+	PCPU_INC(cnt.v_intr);
 
 	ie = isrc->is_event;
 
--- //depot/vendor/freebsd/src/sys/amd64/amd64/trap.c	2007/05/27 19:22:33
+++ //depot/user/attilio/attilio_schedlock/amd64/amd64/trap.c	2007/06/04 10:32:52
@@ -163,7 +163,7 @@
 	register_t addr = 0;
 	ksiginfo_t ksi;
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 	type = frame->tf_trapno;
 
 #ifdef SMP
@@ -737,10 +737,10 @@
 	ksiginfo_t ksi;
 
 	/*
-	 * note: PCPU_LAZY_INC() can only be used if we can afford
+	 * note: PCPU_INC() can only be used if we can afford
 	 * occassional inaccuracy in the count.
 	 */
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 #ifdef DIAGNOSTIC
 	if (ISPL(frame->tf_cs) != SEL_UPL) {
--- //depot/vendor/freebsd/src/sys/amd64/ia32/ia32_syscall.c	2006/12/17 06:52:26
+++ //depot/user/attilio/attilio_schedlock/amd64/ia32/ia32_syscall.c	2007/06/04 10:32:52
@@ -105,10 +105,10 @@
 	ksiginfo_t ksi;
 
 	/*
-	 * note: PCPU_LAZY_INC() can only be used if we can afford
+	 * note: PCPU_INC() can only be used if we can afford
 	 * occassional inaccuracy in the count.
 	 */
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 	td->td_pticks = 0;
 	td->td_frame = frame;
--- //depot/vendor/freebsd/src/sys/amd64/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/amd64/include/pcpu.h	2007/06/04 11:49:41
@@ -56,7 +56,8 @@
 extern struct pcpu *pcpup;
 
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)
-#define	PCPU_LAZY_INC(member)	(++pcpup->pc_ ## member)
+#define	PCPU_ADD(member, val)	(pcpup->pc_ ## member += (val))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
 #define	PCPU_SET(member, val)	(pcpup->pc_ ## member = (val))
 
@@ -110,10 +111,31 @@
 })
 
 /*
+ * Adds the value to the per-cpu counter name.  The implementation
+ * must be atomic with respect to interrupts.
+ */
+#define	__PCPU_ADD(name, val) do {					\
+	__pcpu_type(name) __val;					\
+	struct __s {							\
+		u_char	__b[MIN(sizeof(__pcpu_type(name)), 8)];		\
+	} __s;								\
+									\
+	__val = (val);							\
+	if (sizeof(__val) == 1 || sizeof(__val) == 2 ||			\
+	    sizeof(__val) == 4 || sizeof(__val) == 8) {			\
+		__s = *(struct __s *)(void *)&__val;			\
+		__asm __volatile("add %1,%%gs:%0"			\
+		    : "=m" (*(struct __s *)(__pcpu_offset(name)))	\
+		    : "r" (__s));					\
+	} else								\
+		*__PCPU_PTR(name) += __val;				\
+} while (0)
+
+/*
  * Increments the value of the per-cpu counter name.  The implementation
  * must be atomic with respect to interrupts.
  */
-#define	__PCPU_LAZY_INC(name) do {					\
+#define	__PCPU_INC(name) do {						\
 	CTASSERT(sizeof(__pcpu_type(name)) == 1 ||			\
 	    sizeof(__pcpu_type(name)) == 2 ||				\
 	    sizeof(__pcpu_type(name)) == 4 ||				\
@@ -159,7 +181,8 @@
 }
 
 #define	PCPU_GET(member)	__PCPU_GET(pc_ ## member)
-#define	PCPU_LAZY_INC(member)	__PCPU_LAZY_INC(pc_ ## member)
+#define	PCPU_ADD(member, val)	__PCPU_ADD(pc_ ## member, val)
+#define	PCPU_INC(member)	__PCPU_INC(pc_ ## member)
 #define	PCPU_PTR(member)	__PCPU_PTR(pc_ ## member)
 #define	PCPU_SET(member, val)	__PCPU_SET(pc_ ## member, val)
 
--- //depot/vendor/freebsd/src/sys/arm/arm/intr.c	2007/05/31 19:33:13
+++ //depot/user/attilio/attilio_schedlock/arm/arm/intr.c	2007/06/04 10:32:52
@@ -106,7 +106,7 @@
 	struct thread *td = curthread;
 	int i, thread, ret;
 
-	PCPU_LAZY_INC(cnt.v_intr);
+	PCPU_INC(cnt.v_intr);
 	td->td_intr_nesting_level++;
 	while ((i = arm_get_next_irq()) != -1) {
 		arm_mask_irq(i);
--- //depot/vendor/freebsd/src/sys/arm/arm/trap.c	2007/02/26 05:22:49
+++ //depot/user/attilio/attilio_schedlock/arm/arm/trap.c	2007/06/04 10:32:52
@@ -253,7 +253,7 @@
 	td = curthread;
 	p = td->td_proc;
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 	/* Data abort came from user mode? */
 	user = TRAP_USERMODE(tf);
 
@@ -725,7 +725,7 @@
 	
  	td = curthread;
 	p = td->td_proc;
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	if (TRAP_USERMODE(tf)) {
 		td->td_frame = tf;
@@ -880,7 +880,7 @@
 	register_t *ap, *args, copyargs[MAXARGS];
 	struct sysent *callp;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 	td->td_pticks = 0;
 	if (td->td_ucred != td->td_proc->p_ucred)
 		cred_update_thread(td);
--- //depot/vendor/freebsd/src/sys/arm/arm/undefined.c	2006/10/21 04:27:11
+++ //depot/user/attilio/attilio_schedlock/arm/arm/undefined.c	2007/06/04 10:32:52
@@ -191,7 +191,7 @@
 		enable_interrupts(I32_bit|F32_bit);
 
 	frame->tf_pc -= INSN_SIZE;
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	fault_pc = frame->tf_pc;
 
--- //depot/vendor/freebsd/src/sys/arm/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/arm/include/pcpu.h	2007/06/04 11:49:41
@@ -57,7 +57,8 @@
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
  */
-#define	PCPU_LAZY_INC(member)	(++__pcpu.pc_ ## member)
+#define	PCPU_ADD(member, value)	(__pcpu.pc_ ## member += (value))
+#define	PCPU_INC(member)	PCPU_LAZY_ADD(member, 1)
 #define	PCPU_PTR(member)	(&__pcpu.pc_ ## member)
 #define	PCPU_SET(member,value)	(__pcpu.pc_ ## member = (value))
 
--- //depot/vendor/freebsd/src/sys/i386/i386/intr_machdep.c	2007/05/31 19:28:33
+++ //depot/user/attilio/attilio_schedlock/i386/i386/intr_machdep.c	2007/06/04 10:32:52
@@ -241,7 +241,7 @@
 	 * processed too.
 	 */
 	(*isrc->is_count)++;
-	PCPU_LAZY_INC(cnt.v_intr);
+	PCPU_INC(cnt.v_intr);
 
 	ie = isrc->is_event;
 
@@ -312,7 +312,7 @@
 	 * processed too.
 	 */
 	(*isrc->is_count)++;
-	PCPU_LAZY_INC(cnt.v_intr);
+	PCPU_INC(cnt.v_intr);
 
 	ie = isrc->is_event;
 
--- //depot/vendor/freebsd/src/sys/i386/i386/trap.c	2007/05/27 19:22:33
+++ //depot/user/attilio/attilio_schedlock/i386/i386/trap.c	2007/06/04 10:32:52
@@ -181,7 +181,7 @@
 	static int lastalert = 0;
 #endif
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 	type = frame->tf_trapno;
 
 #ifdef SMP
@@ -922,10 +922,10 @@
 	ksiginfo_t ksi;
 
 	/*
-	 * note: PCPU_LAZY_INC() can only be used if we can afford
+	 * note: PCPU_INC() can only be used if we can afford
 	 * occassional inaccuracy in the count.
 	 */
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 #ifdef DIAGNOSTIC
 	if (ISPL(frame->tf_cs) != SEL_UPL) {
--- //depot/vendor/freebsd/src/sys/i386/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/i386/include/pcpu.h	2007/06/04 11:49:41
@@ -62,7 +62,8 @@
 extern struct pcpu *pcpup;
 
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)
-#define	PCPU_LAZY_INC(member)	(++pcpup->pc_ ## member)
+#define	PCPU_ADD(member, val)	(pcpu->pc_ ## member += (val))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
 #define	PCPU_SET(member, val)	(pcpup->pc_ ## member = (val))
 
@@ -116,10 +117,31 @@
 })
 
 /*
+ * Adds a value of the per-cpu counter name.  The implementation
+ * must be atomic with respect to interrupts.
+ */
+#define	__PCPU_ADD(name, val) do {					\
+	__pcpu_type(name) __val;					\
+	struct __s {							\
+		u_char	__b[MIN(sizeof(__pcpu_type(name)), 4)];		\
+	} __s;								\
+									\
+	__val = (val);							\
+	if (sizeof(__val) == 1 || sizeof(__val) == 2 ||			\
+	    sizeof(__val) == 4) {					\
+		__s = *(struct __s *)(void *)&__val;			\
+		__asm __volatile("add %1,%%fs:%0"			\
+		    : "=m" (*(struct __s *)(__pcpu_offset(name)))	\
+		    : "r" (__s));					\
+	} else								\
+		*__PCPU_PTR(name) += __val;				\
+} while (0)
+
+/*
  * Increments the value of the per-cpu counter name.  The implementation
  * must be atomic with respect to interrupts.
  */
-#define	__PCPU_LAZY_INC(name) do {					\
+#define	__PCPU_INC(name) do {						\
 	CTASSERT(sizeof(__pcpu_type(name)) == 1 ||			\
 	    sizeof(__pcpu_type(name)) == 2 ||				\
 	    sizeof(__pcpu_type(name)) == 4);				\
@@ -160,7 +182,8 @@
 }
 
 #define	PCPU_GET(member)	__PCPU_GET(pc_ ## member)
-#define	PCPU_LAZY_INC(member)	__PCPU_LAZY_INC(pc_ ## member)
+#define	PCPU_ADD(member, val)	__PCPU_ADD(pc_ ## member, val)
+#define	PCPU_INC(member)	__PCPU_INC(pc_ ## member)
 #define	PCPU_PTR(member)	__PCPU_PTR(pc_ ## member)
 #define	PCPU_SET(member, val)	__PCPU_SET(pc_ ## member, val)
 
--- //depot/vendor/freebsd/src/sys/ia64/ia32/ia32_trap.c	2006/09/16 17:05:57
+++ //depot/user/attilio/attilio_schedlock/ia64/ia32/ia32_trap.c	2007/06/04 10:32:52
@@ -64,7 +64,7 @@
 	int error, i, narg;
 	ksiginfo_t ksi;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 	td = curthread;
 	params = (caddr_t)(tf->tf_special.sp & ((1L<<32)-1)) +
@@ -220,7 +220,7 @@
 	KASSERT(TRAPF_USERMODE(tf), ("%s: In kernel mode???", __func__));
 
 	ia64_set_fpsr(IA64_FPSR_DEFAULT);
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	td = curthread;
 	td->td_frame = tf;
--- //depot/vendor/freebsd/src/sys/ia64/ia64/interrupt.c	2007/05/31 19:28:33
+++ //depot/user/attilio/attilio_schedlock/ia64/ia64/interrupt.c	2007/06/04 10:32:52
@@ -154,7 +154,7 @@
 	if (vector == CLOCK_VECTOR) {/* clock interrupt */
 		/* CTR0(KTR_INTR, "clock interrupt"); */
 
-		PCPU_LAZY_INC(cnt.v_intr);
+		PCPU_INC(cnt.v_intr);
 #ifdef EVCNT_COUNTERS
 		clock_intr_evcnt.ev_count++;
 #else
--- //depot/vendor/freebsd/src/sys/ia64/ia64/trap.c	2007/05/21 05:13:00
+++ //depot/user/attilio/attilio_schedlock/ia64/ia64/trap.c	2007/06/04 10:32:52
@@ -363,7 +363,7 @@
 
 	user = TRAPF_USERMODE(tf) ? 1 : 0;
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	td = curthread;
 	p = td->td_proc;
@@ -978,7 +978,7 @@
 	code = tf->tf_scratch.gr15;
 	args = &tf->tf_scratch.gr16;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 	td = curthread;
 	td->td_frame = tf;
--- //depot/vendor/freebsd/src/sys/ia64/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/ia64/include/pcpu.h	2007/06/04 11:32:08
@@ -53,7 +53,8 @@
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
  */
-#define	PCPU_LAZY_INC(member)	(++pcpup->pc_ ## member)
+#define	PCPU_ADD(member, value)	(pcpup->pc_ ## member += (value))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
 #define	PCPU_SET(member,value)	(pcpup->pc_ ## member = (value))
 
--- //depot/vendor/freebsd/src/sys/kern/kern_intr.c	2007/05/31 19:28:33
+++ //depot/user/attilio/attilio_schedlock/kern/kern_intr.c	2007/06/04 10:32:52
@@ -931,7 +931,7 @@
 	atomic_store_rel_int(&ih->ih_need, 1);
 
 	if (!(flags & SWI_DELAY)) {
-		PCPU_LAZY_INC(cnt.v_soft);
+		PCPU_INC(cnt.v_soft);
 #ifdef INTR_FILTER
 		error = intr_event_schedule_thread(ie, ie->ie_thread);
 #else
--- //depot/vendor/freebsd/src/sys/powerpc/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/powerpc/include/pcpu.h	2007/06/04 11:32:08
@@ -62,7 +62,8 @@
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
  */
-#define	PCPU_LAZY_INC(member)	(++PCPUP->pc_ ## member)
+#define	PCPU_ADD(member, value)	(PCPUP->pc_ ## member += (value))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&PCPUP->pc_ ## member)
 #define	PCPU_SET(member,value)	(PCPUP->pc_ ## member = (value))
 
--- //depot/vendor/freebsd/src/sys/powerpc/powerpc/trap.c	2007/03/09 04:07:02
+++ //depot/user/attilio/attilio_schedlock/powerpc/powerpc/trap.c	2007/06/04 10:32:52
@@ -149,7 +149,7 @@
 	u_int		ucode;
 	ksiginfo_t	ksi;
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	td = PCPU_GET(curthread);
 	p = td->td_proc;
@@ -349,7 +349,7 @@
 	td = PCPU_GET(curthread);
 	p = td->td_proc;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 #ifdef KSE
 	if (p->p_flag & P_SA)
--- //depot/vendor/freebsd/src/sys/sparc64/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/sparc64/include/pcpu.h	2007/06/04 11:32:08
@@ -71,7 +71,8 @@
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
  */
-#define	PCPU_LAZY_INC(member)	(++pcpup->pc_ ## member)
+#define	PCPU_ADD(member, value)	(pcpup->pc_ ## member += (value))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
 #define	PCPU_SET(member,value)	(pcpup->pc_ ## member = (value))
 
--- //depot/vendor/freebsd/src/sys/sparc64/sparc64/trap.c	2007/03/09 04:07:02
+++ //depot/user/attilio/attilio_schedlock/sparc64/sparc64/trap.c	2007/06/04 10:32:52
@@ -243,7 +243,7 @@
 	    trap_msg[tf->tf_type & ~T_KERNEL],
 	    (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	if ((tf->tf_tstate & TSTATE_PRIV) == 0) {
 		KASSERT(td != NULL, ("trap: curthread NULL"));
@@ -518,7 +518,7 @@
 
 	p = td->td_proc;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 	narg = 0;
 	error = 0;
--- //depot/vendor/freebsd/src/sys/sun4v/include/pcpu.h	2007/03/11 05:57:05
+++ //depot/user/attilio/attilio_schedlock/sun4v/include/pcpu.h	2007/06/04 11:32:08
@@ -92,7 +92,8 @@
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
  */
-#define	PCPU_LAZY_INC(member)	(++pcpup->pc_ ## member)
+#define	PCPU_ADD(member, value)	(pcpup->pc_ ## member += (value))
+#define	PCPU_INC(member)	PCPU_ADD(member, 1)
 #define	PCPU_PTR(member)	(&pcpup->pc_ ## member)
 #define	PCPU_SET(member,value)	(pcpup->pc_ ## member = (value))
 
--- //depot/vendor/freebsd/src/sys/sun4v/sun4v/trap.c	2007/05/25 01:27:53
+++ //depot/user/attilio/attilio_schedlock/sun4v/sun4v/trap.c	2007/06/04 10:32:52
@@ -268,7 +268,7 @@
 	    trap_msg[trapno],
 	    (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));
 
-	PCPU_LAZY_INC(cnt.v_trap);
+	PCPU_INC(cnt.v_trap);
 
 	trapno = (type & TRAP_MASK);
 	ctx = (type >> TRAP_CTX_SHIFT);
@@ -575,7 +575,7 @@
 
 	p = td->td_proc;
 
-	PCPU_LAZY_INC(cnt.v_syscall);
+	PCPU_INC(cnt.v_syscall);
 
 	narg = 0;
 	error = 0;
--- //depot/vendor/freebsd/src/sys/vm/vm_fault.c	2007/06/01 01:13:08
+++ //depot/user/attilio/attilio_schedlock/vm/vm_fault.c	2007/06/04 10:32:52
@@ -219,7 +219,7 @@
 
 	hardfault = 0;
 	growstack = TRUE;
-	PCPU_LAZY_INC(cnt.v_vm_faults);
+	PCPU_INC(cnt.v_vm_faults);
 
 RetryFault:;
 
@@ -394,7 +394,7 @@
 				}
 				vm_object_pip_wakeup(fs.object);
 				VM_OBJECT_UNLOCK(fs.object);
-				PCPU_LAZY_INC(cnt.v_intrans);
+				PCPU_INC(cnt.v_intrans);
 				vm_object_deallocate(fs.first_object);
 				goto RetryFault;
 			}
@@ -668,9 +668,9 @@
 			if ((fs.m->flags & PG_ZERO) == 0) {
 				pmap_zero_page(fs.m);
 			} else {
-				PCPU_LAZY_INC(cnt.v_ozfod);
+				PCPU_INC(cnt.v_ozfod);
 			}
-			PCPU_LAZY_INC(cnt.v_zfod);
+			PCPU_INC(cnt.v_zfod);
 			fs.m->valid = VM_PAGE_BITS_ALL;
 			break;	/* break to PAGE HAS BEEN FOUND */
 		} else {
@@ -752,7 +752,7 @@
 				vm_page_busy(fs.m);
 				fs.first_m = fs.m;
 				fs.m = NULL;
-				PCPU_LAZY_INC(cnt.v_cow_optim);
+				PCPU_INC(cnt.v_cow_optim);
 			} else {
 				/*
 				 * Oh, well, lets copy it.
@@ -780,7 +780,7 @@
 			fs.m = fs.first_m;
 			if (!is_first_object_locked)
 				VM_OBJECT_LOCK(fs.object);
-			PCPU_LAZY_INC(cnt.v_cow_faults);
+			PCPU_INC(cnt.v_cow_faults);
 		} else {
 			prot &= ~VM_PROT_WRITE;
 		}
