Update six locks

include/linux/six.h

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 
 #ifndef _LINUX_SIX_H
 #define _LINUX_SIX_H
@@ -50,12 +50,12 @@
  *   six_trylock_convert(lock, from, to)
  *
  * A lock may be held multiple types by the same thread (for read or intent,
- * not write) - up to SIX_LOCK_MAX_RECURSE. However, the six locks code does
+ * not write). However, the six locks code does _not_ implement the actual
- * _not_ implement the actual recursive checks itself though - rather, if your
+ * recursive checks itself though - rather, if your code (e.g. btree iterator
- * code (e.g. btree iterator code) knows that the current thread already has a
+ * code) knows that the current thread already has a lock held, and for the
- * lock held, and for the correct type, six_lock_increment() may be used to
+ * correct type, six_lock_increment() may be used to bump up the counter for
- * bump up the counter for that type - the only effect is that one more call to
+ * that type - the only effect is that one more call to unlock will be required
- * unlock will be required before the lock is unlocked.
+ * before the lock is unlocked.
  */
 
 #include <linux/lockdep.h>
@@ -80,8 +80,8 @@ union six_lock_state {
 	};
 
 	struct {
-		unsigned	read_lock:26;
+		unsigned	read_lock:28;
-		unsigned	intent_lock:3;
+		unsigned	intent_lock:1;
 		unsigned	waiters:3;
 		/*
 		 * seq works much like in seqlocks: it's incremented every time
@@ -96,8 +96,6 @@ union six_lock_state {
 	};
 };
 
-#define SIX_LOCK_MAX_RECURSE	((1 << 3) - 1)
-
 enum six_lock_type {
 	SIX_LOCK_read,
 	SIX_LOCK_intent,
@@ -106,6 +104,7 @@ enum six_lock_type {
 
 struct six_lock {
 	union six_lock_state	state;
+	unsigned		intent_lock_recurse;
 	struct task_struct	*owner;
 	struct optimistic_spin_queue osq;
 
@@ -139,8 +138,6 @@ do {									\
 
 #define __SIX_VAL(field, _v)	(((union six_lock_state) { .field = _v }).v)
 
-#ifdef SIX_LOCK_SEPARATE_LOCKFNS
-
 #define __SIX_LOCK(type)						\
 bool six_trylock_##type(struct six_lock *);				\
 bool six_relock_##type(struct six_lock *, u32);				\
@@ -185,41 +182,6 @@ static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type typ
 	SIX_LOCK_DISPATCH(type, six_unlock, lock);
 }
 
-#else
-
-bool six_trylock_type(struct six_lock *, enum six_lock_type);
-bool six_relock_type(struct six_lock *, enum six_lock_type, unsigned);
-void six_lock_type(struct six_lock *, enum six_lock_type);
-void six_unlock_type(struct six_lock *, enum six_lock_type);
-
-#define __SIX_LOCK(type)						\
-static __always_inline bool six_trylock_##type(struct six_lock *lock)	\
-{									\
-	return six_trylock_type(lock, SIX_LOCK_##type);			\
-}									\
-									\
-static __always_inline bool six_relock_##type(struct six_lock *lock, u32 seq)\
-{									\
-	return six_relock_type(lock, SIX_LOCK_##type, seq);		\
-}									\
-									\
-static __always_inline void six_lock_##type(struct six_lock *lock)	\
-{									\
-	six_lock_type(lock, SIX_LOCK_##type);				\
-}									\
-									\
-static __always_inline void six_unlock_##type(struct six_lock *lock)	\
-{									\
-	six_unlock_type(lock, SIX_LOCK_##type);				\
-}
-
-__SIX_LOCK(read)
-__SIX_LOCK(intent)
-__SIX_LOCK(write)
-#undef __SIX_LOCK
-
-#endif
-
 void six_lock_downgrade(struct six_lock *);
 bool six_lock_tryupgrade(struct six_lock *);
 bool six_trylock_convert(struct six_lock *, enum six_lock_type,

linux/six.c

@@ -76,17 +76,6 @@ static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type,
 	}
 }
 
-static inline void six_clear_owner(struct six_lock *lock, enum six_lock_type type)
-{
-	if (type != SIX_LOCK_intent)
-		return;
-
-	EBUG_ON(lock->owner != current);
-
-	if (lock->state.intent_lock == 1)
-		lock->owner = NULL;
-}
-
 static __always_inline bool do_six_trylock_type(struct six_lock *lock,
 						enum six_lock_type type)
 {
@@ -393,16 +382,24 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 	EBUG_ON(type == SIX_LOCK_write &&
 		!(lock->state.v & __SIX_LOCK_HELD_intent));
 
-	six_clear_owner(lock, type);
+	six_release(&lock->dep_map);
+
+	if (type == SIX_LOCK_intent) {
+		EBUG_ON(lock->owner != current);
+
+		if (lock->intent_lock_recurse) {
+			--lock->intent_lock_recurse;
+			return;
+		}
+
+		lock->owner = NULL;
+	}
 
 	state.v = atomic64_add_return_release(l[type].unlock_val,
 					      &lock->state.counter);
-	six_release(&lock->dep_map);
 	six_lock_wakeup(lock, state, l[type].unlock_wakeup);
 }
 
-#ifdef SIX_LOCK_SEPARATE_LOCKFNS
-
 #define __SIX_LOCK(type)						\
 bool six_trylock_##type(struct six_lock *lock)				\
 {									\
@@ -434,36 +431,6 @@ __SIX_LOCK(write)
 
 #undef __SIX_LOCK
 
-#else
-
-bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
-{
-	return __six_trylock_type(lock, type);
-}
-EXPORT_SYMBOL_GPL(six_trylock_type);
-
-bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
-		     unsigned seq)
-{
-	return __six_relock_type(lock, type, seq);
-
-}
-EXPORT_SYMBOL_GPL(six_relock_type);
-
-void six_lock_type(struct six_lock *lock, enum six_lock_type type)
-{
-	__six_lock_type(lock, type);
-}
-EXPORT_SYMBOL_GPL(six_lock_type);
-
-void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
-{
-	__six_unlock_type(lock, type);
-}
-EXPORT_SYMBOL_GPL(six_unlock_type);
-
-#endif
-
 /* Convert from intent to read: */
 void six_lock_downgrade(struct six_lock *lock)
 {
@@ -530,6 +497,16 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 
 	/* XXX: assert already locked, and that we don't overflow: */
 
-	atomic64_add(l[type].lock_val, &lock->state.counter);
+	switch (type) {
+	case SIX_LOCK_read:
+		atomic64_add(l[type].lock_val, &lock->state.counter);
+		break;
+	case SIX_LOCK_intent:
+		lock->intent_lock_recurse++;
+		break;
+	case SIX_LOCK_write:
+		BUG();
+		break;
+	}
 }
 EXPORT_SYMBOL_GPL(six_lock_increment);