diff --git a/include/linux/six.h b/include/linux/six.h index 40e213f2..0fb1b2f4 100644 --- a/include/linux/six.h +++ b/include/linux/six.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_SIX_H #define _LINUX_SIX_H @@ -50,12 +50,12 @@ * six_trylock_convert(lock, from, to) * * A lock may be held multiple types by the same thread (for read or intent, - * not write) - up to SIX_LOCK_MAX_RECURSE. However, the six locks code does - * _not_ implement the actual recursive checks itself though - rather, if your - * code (e.g. btree iterator code) knows that the current thread already has a - * lock held, and for the correct type, six_lock_increment() may be used to - * bump up the counter for that type - the only effect is that one more call to - * unlock will be required before the lock is unlocked. + * not write). However, the six locks code does _not_ implement the actual + * recursive checks itself though - rather, if your code (e.g. btree iterator + * code) knows that the current thread already has a lock held, and for the + * correct type, six_lock_increment() may be used to bump up the counter for + * that type - the only effect is that one more call to unlock will be required + * before the lock is unlocked. */ #include @@ -80,8 +80,8 @@ union six_lock_state { }; struct { - unsigned read_lock:26; - unsigned intent_lock:3; + unsigned read_lock:28; + unsigned intent_lock:1; unsigned waiters:3; /* * seq works much like in seqlocks: it's incremented every time @@ -96,8 +96,6 @@ union six_lock_state { }; }; -#define SIX_LOCK_MAX_RECURSE ((1 << 3) - 1) - enum six_lock_type { SIX_LOCK_read, SIX_LOCK_intent, @@ -106,6 +104,7 @@ enum six_lock_type { struct six_lock { union six_lock_state state; + unsigned intent_lock_recurse; struct task_struct *owner; struct optimistic_spin_queue osq; @@ -139,8 +138,6 @@ do { \ #define __SIX_VAL(field, _v) (((union six_lock_state) { .field = _v }).v) -#ifdef SIX_LOCK_SEPARATE_LOCKFNS - #define __SIX_LOCK(type) \ bool six_trylock_##type(struct six_lock *); \ bool six_relock_##type(struct six_lock *, u32); \ @@ -185,41 +182,6 @@ static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type typ SIX_LOCK_DISPATCH(type, six_unlock, lock); } -#else - -bool six_trylock_type(struct six_lock *, enum six_lock_type); -bool six_relock_type(struct six_lock *, enum six_lock_type, unsigned); -void six_lock_type(struct six_lock *, enum six_lock_type); -void six_unlock_type(struct six_lock *, enum six_lock_type); - -#define __SIX_LOCK(type) \ -static __always_inline bool six_trylock_##type(struct six_lock *lock) \ -{ \ - return six_trylock_type(lock, SIX_LOCK_##type); \ -} \ - \ -static __always_inline bool six_relock_##type(struct six_lock *lock, u32 seq)\ -{ \ - return six_relock_type(lock, SIX_LOCK_##type, seq); \ -} \ - \ -static __always_inline void six_lock_##type(struct six_lock *lock) \ -{ \ - six_lock_type(lock, SIX_LOCK_##type); \ -} \ - \ -static __always_inline void six_unlock_##type(struct six_lock *lock) \ -{ \ - six_unlock_type(lock, SIX_LOCK_##type); \ -} - -__SIX_LOCK(read) -__SIX_LOCK(intent) -__SIX_LOCK(write) -#undef __SIX_LOCK - -#endif - void six_lock_downgrade(struct six_lock *); bool six_lock_tryupgrade(struct six_lock *); bool six_trylock_convert(struct six_lock *, enum six_lock_type, diff --git a/linux/six.c b/linux/six.c index aceeabb0..9fa58b6f 100644 --- a/linux/six.c +++ b/linux/six.c @@ -76,17 +76,6 @@ static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type, } } -static inline void six_clear_owner(struct six_lock *lock, enum six_lock_type type) -{ - if (type != SIX_LOCK_intent) - return; - - EBUG_ON(lock->owner != current); - - if (lock->state.intent_lock == 1) - lock->owner = NULL; -} - static __always_inline bool do_six_trylock_type(struct six_lock *lock, enum six_lock_type type) { @@ -393,16 +382,24 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type) EBUG_ON(type == SIX_LOCK_write && !(lock->state.v & __SIX_LOCK_HELD_intent)); - six_clear_owner(lock, type); + six_release(&lock->dep_map); + + if (type == SIX_LOCK_intent) { + EBUG_ON(lock->owner != current); + + if (lock->intent_lock_recurse) { + --lock->intent_lock_recurse; + return; + } + + lock->owner = NULL; + } state.v = atomic64_add_return_release(l[type].unlock_val, &lock->state.counter); - six_release(&lock->dep_map); six_lock_wakeup(lock, state, l[type].unlock_wakeup); } -#ifdef SIX_LOCK_SEPARATE_LOCKFNS - #define __SIX_LOCK(type) \ bool six_trylock_##type(struct six_lock *lock) \ { \ @@ -434,36 +431,6 @@ __SIX_LOCK(write) #undef __SIX_LOCK -#else - -bool six_trylock_type(struct six_lock *lock, enum six_lock_type type) -{ - return __six_trylock_type(lock, type); -} -EXPORT_SYMBOL_GPL(six_trylock_type); - -bool six_relock_type(struct six_lock *lock, enum six_lock_type type, - unsigned seq) -{ - return __six_relock_type(lock, type, seq); - -} -EXPORT_SYMBOL_GPL(six_relock_type); - -void six_lock_type(struct six_lock *lock, enum six_lock_type type) -{ - __six_lock_type(lock, type); -} -EXPORT_SYMBOL_GPL(six_lock_type); - -void six_unlock_type(struct six_lock *lock, enum six_lock_type type) -{ - __six_unlock_type(lock, type); -} -EXPORT_SYMBOL_GPL(six_unlock_type); - -#endif - /* Convert from intent to read: */ void six_lock_downgrade(struct six_lock *lock) { @@ -530,6 +497,16 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type) /* XXX: assert already locked, and that we don't overflow: */ - atomic64_add(l[type].lock_val, &lock->state.counter); + switch (type) { + case SIX_LOCK_read: + atomic64_add(l[type].lock_val, &lock->state.counter); + break; + case SIX_LOCK_intent: + lock->intent_lock_recurse++; + break; + case SIX_LOCK_write: + BUG(); + break; + } } EXPORT_SYMBOL_GPL(six_lock_increment);