Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

[cascardo/linux.git] / kernel / locking / qrwlock.c

diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index f956ede..6c5da48 100644 (file)
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -1,5 +1,5 @@
 /*
- * Queue read/write lock
+ * Queued read/write locks
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -22,6 +22,26 @@
 #include <linux/hardirq.h>
 #include <asm/qrwlock.h>
 
+/*
+ * This internal data structure is used for optimizing access to some of
+ * the subfields within the atomic_t cnts.
+ */
+struct __qrwlock {
+       union {
+               atomic_t cnts;
+               struct {
+#ifdef __LITTLE_ENDIAN
+                       u8 wmode;       /* Writer mode   */
+                       u8 rcnts[3];    /* Reader counts */
+#else
+                       u8 rcnts[3];    /* Reader counts */
+                       u8 wmode;       /* Writer mode   */
+#endif
+               };
+       };
+       arch_spinlock_t lock;
+};
+
 /**
  * rspin_until_writer_unlock - inc reader count & spin until writer is gone
  * @lock  : Pointer to queue rwlock structure
@@ -107,10 +127,10 @@ void queue_write_lock_slowpath(struct qrwlock *lock)
         * or wait for a previous writer to go away.
         */
        for (;;) {
-               cnts = atomic_read(&lock->cnts);
-               if (!(cnts & _QW_WMASK) &&
-                   (atomic_cmpxchg(&lock->cnts, cnts,
-                                   cnts | _QW_WAITING) == cnts))
+               struct __qrwlock *l = (struct __qrwlock *)lock;
+
+               if (!READ_ONCE(l->wmode) &&
+                  (cmpxchg(&l->wmode, 0, _QW_WAITING) == 0))
                        break;
 
                cpu_relax_lowlatency();