rtmutex.c 47.3 KB
Newer Older
I
Ingo Molnar 已提交
1 2 3 4 5 6 7 8 9
/*
 * RT-Mutexes: simple blocking mutual exclusion locks with PI support
 *
 * started by Ingo Molnar and Thomas Gleixner.
 *
 *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
 *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
 *  Copyright (C) 2006 Esben Nielsen
10
 *
11
 *  See Documentation/locking/rt-mutex-design.txt for details.
I
Ingo Molnar 已提交
12 13
 */
#include <linux/spinlock.h>
14
#include <linux/export.h>
15
#include <linux/sched/signal.h>
16
#include <linux/sched/rt.h>
17
#include <linux/sched/deadline.h>
18
#include <linux/sched/wake_q.h>
19
#include <linux/sched/debug.h>
I
Ingo Molnar 已提交
20 21 22 23 24 25 26
#include <linux/timer.h>

#include "rtmutex_common.h"

/*
 * lock->owner state tracking:
 *
27 28
 * lock->owner holds the task_struct pointer of the owner. Bit 0
 * is used to keep track of the "lock has waiters" state.
I
Ingo Molnar 已提交
29
 *
30 31 32 33 34 35
 * owner	bit0
 * NULL		0	lock is free (fast acquire possible)
 * NULL		1	lock is free and has waiters and the top waiter
 *				is going to take the lock*
 * taskpointer	0	lock is held (fast release possible)
 * taskpointer	1	lock is held and has waiters**
I
Ingo Molnar 已提交
36 37
 *
 * The fast atomic compare exchange based acquire and release is only
38 39 40 41 42 43
 * possible when bit 0 of lock->owner is 0.
 *
 * (*) It also can be a transitional state when grabbing the lock
 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
 * we need to set the bit0 before looking at the lock, and the owner may be
 * NULL in this small time, hence this can be a transitional state.
I
Ingo Molnar 已提交
44
 *
45 46 47 48
 * (**) There is a small time when bit 0 is set but there are no
 * waiters. This can happen when grabbing the lock in the slow path.
 * To prevent a cmpxchg of the owner releasing the lock, we need to
 * set this bit before looking at the lock.
I
Ingo Molnar 已提交
49 50
 */

51
static void
52
rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
I
Ingo Molnar 已提交
53
{
54
	unsigned long val = (unsigned long)owner;
I
Ingo Molnar 已提交
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69

	if (rt_mutex_has_waiters(lock))
		val |= RT_MUTEX_HAS_WAITERS;

	lock->owner = (struct task_struct *)val;
}

static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
{
	lock->owner = (struct task_struct *)
			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
}

static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
{
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
	unsigned long owner, *p = (unsigned long *) &lock->owner;

	if (rt_mutex_has_waiters(lock))
		return;

	/*
	 * The rbtree has no waiters enqueued, now make sure that the
	 * lock->owner still has the waiters bit set, otherwise the
	 * following can happen:
	 *
	 * CPU 0	CPU 1		CPU2
	 * l->owner=T1
	 *		rt_mutex_lock(l)
	 *		lock(l->lock)
	 *		l->owner = T1 | HAS_WAITERS;
	 *		enqueue(T2)
	 *		boost()
	 *		  unlock(l->lock)
	 *		block()
	 *
	 *				rt_mutex_lock(l)
	 *				lock(l->lock)
	 *				l->owner = T1 | HAS_WAITERS;
	 *				enqueue(T3)
	 *				boost()
	 *				  unlock(l->lock)
	 *				block()
	 *		signal(->T2)	signal(->T3)
	 *		lock(l->lock)
	 *		dequeue(T2)
	 *		deboost()
	 *		  unlock(l->lock)
	 *				lock(l->lock)
	 *				dequeue(T3)
	 *				 ==> wait list is empty
	 *				deboost()
	 *				 unlock(l->lock)
	 *		lock(l->lock)
	 *		fixup_rt_mutex_waiters()
	 *		  if (wait_list_empty(l) {
	 *		    l->owner = owner
	 *		    owner = l->owner & ~HAS_WAITERS;
	 *		      ==> l->owner = T1
	 *		  }
	 *				lock(l->lock)
	 * rt_mutex_unlock(l)		fixup_rt_mutex_waiters()
	 *				  if (wait_list_empty(l) {
	 *				    owner = l->owner & ~HAS_WAITERS;
	 * cmpxchg(l->owner, T1, NULL)
	 *  ===> Success (l->owner = NULL)
	 *
	 *				    l->owner = owner
	 *				      ==> l->owner = T1
	 *				  }
	 *
	 * With the check for the waiter bit in place T3 on CPU2 will not
	 * overwrite. All tasks fiddling with the waiters bit are
	 * serialized by l->lock, so nothing else can modify the waiters
	 * bit. If the bit is set then nothing can change l->owner either
	 * so the simple RMW is safe. The cmpxchg() will simply fail if it
	 * happens in the middle of the RMW because the waiters bit is
	 * still set.
	 */
	owner = READ_ONCE(*p);
	if (owner & RT_MUTEX_HAS_WAITERS)
		WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
I
Ingo Molnar 已提交
136 137
}

138
/*
139 140
 * We can speed up the acquire/release, if there's no debugging state to be
 * set up.
141
 */
142
#ifndef CONFIG_DEBUG_RT_MUTEXES
143 144 145 146 147 148 149 150 151
# define rt_mutex_cmpxchg_relaxed(l,c,n) (cmpxchg_relaxed(&l->owner, c, n) == c)
# define rt_mutex_cmpxchg_acquire(l,c,n) (cmpxchg_acquire(&l->owner, c, n) == c)
# define rt_mutex_cmpxchg_release(l,c,n) (cmpxchg_release(&l->owner, c, n) == c)

/*
 * Callers must hold the ->wait_lock -- which is the whole purpose as we force
 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
 * relaxed semantics suffice.
 */
152 153 154 155 156 157
static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
{
	unsigned long owner, *p = (unsigned long *) &lock->owner;

	do {
		owner = *p;
158 159
	} while (cmpxchg_relaxed(p, owner,
				 owner | RT_MUTEX_HAS_WAITERS) != owner);
160
}
T
Thomas Gleixner 已提交
161 162 163 164 165 166 167

/*
 * Safe fastpath aware unlock:
 * 1) Clear the waiters bit
 * 2) Drop lock->wait_lock
 * 3) Try to unlock the lock with cmpxchg
 */
168 169
static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
					unsigned long flags)
T
Thomas Gleixner 已提交
170 171 172 173 174
	__releases(lock->wait_lock)
{
	struct task_struct *owner = rt_mutex_owner(lock);

	clear_rt_mutex_waiters(lock);
175
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
T
Thomas Gleixner 已提交
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
	/*
	 * If a new waiter comes in between the unlock and the cmpxchg
	 * we have two situations:
	 *
	 * unlock(wait_lock);
	 *					lock(wait_lock);
	 * cmpxchg(p, owner, 0) == owner
	 *					mark_rt_mutex_waiters(lock);
	 *					acquire(lock);
	 * or:
	 *
	 * unlock(wait_lock);
	 *					lock(wait_lock);
	 *					mark_rt_mutex_waiters(lock);
	 *
	 * cmpxchg(p, owner, 0) != owner
	 *					enqueue_waiter();
	 *					unlock(wait_lock);
	 * lock(wait_lock);
	 * wake waiter();
	 * unlock(wait_lock);
	 *					lock(wait_lock);
	 *					acquire(lock);
	 */
200
	return rt_mutex_cmpxchg_release(lock, owner, NULL);
T
Thomas Gleixner 已提交
201 202
}

203
#else
204 205 206 207
# define rt_mutex_cmpxchg_relaxed(l,c,n)	(0)
# define rt_mutex_cmpxchg_acquire(l,c,n)	(0)
# define rt_mutex_cmpxchg_release(l,c,n)	(0)

208 209 210 211 212
static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
{
	lock->owner = (struct task_struct *)
			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
}
T
Thomas Gleixner 已提交
213 214 215 216

/*
 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
 */
217 218
static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
					unsigned long flags)
T
Thomas Gleixner 已提交
219 220 221
	__releases(lock->wait_lock)
{
	lock->owner = NULL;
222
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
T
Thomas Gleixner 已提交
223 224
	return true;
}
225 226
#endif

227 228 229 230
static inline int
rt_mutex_waiter_less(struct rt_mutex_waiter *left,
		     struct rt_mutex_waiter *right)
{
231
	if (left->prio < right->prio)
232 233 234
		return 1;

	/*
235 236 237 238
	 * If both waiters have dl_prio(), we check the deadlines of the
	 * associated tasks.
	 * If left waiter has a dl_prio(), and we didn't return 1 above,
	 * then right waiter has a dl_prio() too.
239
	 */
240
	if (dl_prio(left->prio))
241 242
		return dl_time_before(left->task->dl.deadline,
				      right->task->dl.deadline);
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324

	return 0;
}

static void
rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
{
	struct rb_node **link = &lock->waiters.rb_node;
	struct rb_node *parent = NULL;
	struct rt_mutex_waiter *entry;
	int leftmost = 1;

	while (*link) {
		parent = *link;
		entry = rb_entry(parent, struct rt_mutex_waiter, tree_entry);
		if (rt_mutex_waiter_less(waiter, entry)) {
			link = &parent->rb_left;
		} else {
			link = &parent->rb_right;
			leftmost = 0;
		}
	}

	if (leftmost)
		lock->waiters_leftmost = &waiter->tree_entry;

	rb_link_node(&waiter->tree_entry, parent, link);
	rb_insert_color(&waiter->tree_entry, &lock->waiters);
}

static void
rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
{
	if (RB_EMPTY_NODE(&waiter->tree_entry))
		return;

	if (lock->waiters_leftmost == &waiter->tree_entry)
		lock->waiters_leftmost = rb_next(&waiter->tree_entry);

	rb_erase(&waiter->tree_entry, &lock->waiters);
	RB_CLEAR_NODE(&waiter->tree_entry);
}

static void
rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
{
	struct rb_node **link = &task->pi_waiters.rb_node;
	struct rb_node *parent = NULL;
	struct rt_mutex_waiter *entry;
	int leftmost = 1;

	while (*link) {
		parent = *link;
		entry = rb_entry(parent, struct rt_mutex_waiter, pi_tree_entry);
		if (rt_mutex_waiter_less(waiter, entry)) {
			link = &parent->rb_left;
		} else {
			link = &parent->rb_right;
			leftmost = 0;
		}
	}

	if (leftmost)
		task->pi_waiters_leftmost = &waiter->pi_tree_entry;

	rb_link_node(&waiter->pi_tree_entry, parent, link);
	rb_insert_color(&waiter->pi_tree_entry, &task->pi_waiters);
}

static void
rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
{
	if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
		return;

	if (task->pi_waiters_leftmost == &waiter->pi_tree_entry)
		task->pi_waiters_leftmost = rb_next(&waiter->pi_tree_entry);

	rb_erase(&waiter->pi_tree_entry, &task->pi_waiters);
	RB_CLEAR_NODE(&waiter->pi_tree_entry);
}

325
static void rt_mutex_adjust_prio(struct task_struct *p)
326
{
327
	struct task_struct *pi_task = NULL;
328

329
	lockdep_assert_held(&p->pi_lock);
330

331 332
	if (task_has_pi_waiters(p))
		pi_task = task_top_pi_waiter(p)->task;
333

334
	rt_mutex_setprio(p, pi_task);
I
Ingo Molnar 已提交
335 336
}

337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
/*
 * Deadlock detection is conditional:
 *
 * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
 * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
 *
 * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
 * conducted independent of the detect argument.
 *
 * If the waiter argument is NULL this indicates the deboost path and
 * deadlock detection is disabled independent of the detect argument
 * and the config settings.
 */
static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
					  enum rtmutex_chainwalk chwalk)
{
	/*
	 * This is just a wrapper function for the following call,
	 * because debug_rt_mutex_detect_deadlock() smells like a magic
	 * debug feature and I wanted to keep the cond function in the
	 * main source file along with the comments instead of having
	 * two of the same in the headers.
	 */
	return debug_rt_mutex_detect_deadlock(waiter, chwalk);
}

I
Ingo Molnar 已提交
363 364 365 366 367
/*
 * Max number of times we'll walk the boosting chain:
 */
int max_lock_depth = 1024;

368 369 370 371 372
static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
{
	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
}

I
Ingo Molnar 已提交
373 374 375
/*
 * Adjust the priority chain. Also used for deadlock detection.
 * Decreases task's usage by one - may thus free the task.
376
 *
377 378
 * @task:	the task owning the mutex (owner) for which a chain walk is
 *		probably needed
379
 * @chwalk:	do we have to carry out deadlock detection?
380 381 382 383 384 385
 * @orig_lock:	the mutex (can be NULL if we are walking the chain to recheck
 *		things for a task that has just got its priority adjusted, and
 *		is waiting on a mutex)
 * @next_lock:	the mutex on which the owner of @orig_lock was blocked before
 *		we dropped its pi_lock. Is never dereferenced, only used for
 *		comparison to detect lock chain changes.
386
 * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
387 388 389 390
 *		its priority to the mutex owner (can be NULL in the case
 *		depicted above or if the top waiter is gone away and we are
 *		actually deboosting the owner)
 * @top_task:	the current top waiter
391
 *
I
Ingo Molnar 已提交
392
 * Returns 0 or -EDEADLK.
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
 *
 * Chain walk basics and protection scope
 *
 * [R] refcount on task
 * [P] task->pi_lock held
 * [L] rtmutex->wait_lock held
 *
 * Step	Description				Protected by
 *	function arguments:
 *	@task					[R]
 *	@orig_lock if != NULL			@top_task is blocked on it
 *	@next_lock				Unprotected. Cannot be
 *						dereferenced. Only used for
 *						comparison.
 *	@orig_waiter if != NULL			@top_task is blocked on it
 *	@top_task				current, or in case of proxy
 *						locking protected by calling
 *						code
 *	again:
 *	  loop_sanity_check();
 *	retry:
 * [1]	  lock(task->pi_lock);			[R] acquire [P]
 * [2]	  waiter = task->pi_blocked_on;		[P]
 * [3]	  check_exit_conditions_1();		[P]
 * [4]	  lock = waiter->lock;			[P]
 * [5]	  if (!try_lock(lock->wait_lock)) {	[P] try to acquire [L]
 *	    unlock(task->pi_lock);		release [P]
 *	    goto retry;
 *	  }
 * [6]	  check_exit_conditions_2();		[P] + [L]
 * [7]	  requeue_lock_waiter(lock, waiter);	[P] + [L]
 * [8]	  unlock(task->pi_lock);		release [P]
 *	  put_task_struct(task);		release [R]
 * [9]	  check_exit_conditions_3();		[L]
 * [10]	  task = owner(lock);			[L]
 *	  get_task_struct(task);		[L] acquire [R]
 *	  lock(task->pi_lock);			[L] acquire [P]
 * [11]	  requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
 * [12]	  check_exit_conditions_4();		[P] + [L]
 * [13]	  unlock(task->pi_lock);		release [P]
 *	  unlock(lock->wait_lock);		release [L]
 *	  goto again;
I
Ingo Molnar 已提交
435
 */
436
static int rt_mutex_adjust_prio_chain(struct task_struct *task,
437
				      enum rtmutex_chainwalk chwalk,
438
				      struct rt_mutex *orig_lock,
439
				      struct rt_mutex *next_lock,
440 441
				      struct rt_mutex_waiter *orig_waiter,
				      struct task_struct *top_task)
I
Ingo Molnar 已提交
442 443
{
	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
444
	struct rt_mutex_waiter *prerequeue_top_waiter;
445
	int ret = 0, depth = 0;
446
	struct rt_mutex *lock;
447
	bool detect_deadlock;
448
	bool requeue = true;
I
Ingo Molnar 已提交
449

450
	detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
I
Ingo Molnar 已提交
451 452 453 454 455 456 457 458

	/*
	 * The (de)boosting is a step by step approach with a lot of
	 * pitfalls. We want this to be preemptible and we want hold a
	 * maximum of two locks per step. So we have to check
	 * carefully whether things change under us.
	 */
 again:
459 460 461
	/*
	 * We limit the lock chain length for each invocation.
	 */
I
Ingo Molnar 已提交
462 463 464 465 466 467 468 469 470 471 472
	if (++depth > max_lock_depth) {
		static int prev_max;

		/*
		 * Print this only once. If the admin changes the limit,
		 * print a new message when reaching the limit again.
		 */
		if (prev_max != max_lock_depth) {
			prev_max = max_lock_depth;
			printk(KERN_WARNING "Maximum lock depth %d reached "
			       "task: %s (%d)\n", max_lock_depth,
473
			       top_task->comm, task_pid_nr(top_task));
I
Ingo Molnar 已提交
474 475 476
		}
		put_task_struct(task);

477
		return -EDEADLK;
I
Ingo Molnar 已提交
478
	}
479 480 481 482 483 484 485

	/*
	 * We are fully preemptible here and only hold the refcount on
	 * @task. So everything can have changed under us since the
	 * caller or our own code below (goto retry/again) dropped all
	 * locks.
	 */
I
Ingo Molnar 已提交
486 487
 retry:
	/*
488
	 * [1] Task cannot go away as we did a get_task() before !
I
Ingo Molnar 已提交
489
	 */
490
	raw_spin_lock_irq(&task->pi_lock);
I
Ingo Molnar 已提交
491

492 493 494
	/*
	 * [2] Get the waiter on which @task is blocked on.
	 */
I
Ingo Molnar 已提交
495
	waiter = task->pi_blocked_on;
496 497 498 499 500

	/*
	 * [3] check_exit_conditions_1() protected by task->pi_lock.
	 */

I
Ingo Molnar 已提交
501 502 503 504 505
	/*
	 * Check whether the end of the boosting chain has been
	 * reached or the state of the chain has changed while we
	 * dropped the locks.
	 */
506
	if (!waiter)
I
Ingo Molnar 已提交
507 508
		goto out_unlock_pi;

509 510
	/*
	 * Check the orig_waiter state. After we dropped the locks,
511
	 * the previous owner of the lock might have released the lock.
512
	 */
513
	if (orig_waiter && !rt_mutex_owner(orig_lock))
514 515
		goto out_unlock_pi;

516 517 518 519 520 521 522 523 524 525 526 527
	/*
	 * We dropped all locks after taking a refcount on @task, so
	 * the task might have moved on in the lock chain or even left
	 * the chain completely and blocks now on an unrelated lock or
	 * on @orig_lock.
	 *
	 * We stored the lock on which @task was blocked in @next_lock,
	 * so we can detect the chain change.
	 */
	if (next_lock != waiter->lock)
		goto out_unlock_pi;

528 529 530 531 532
	/*
	 * Drop out, when the task has no waiters. Note,
	 * top_waiter can be NULL, when we are in the deboosting
	 * mode!
	 */
533 534 535 536 537
	if (top_waiter) {
		if (!task_has_pi_waiters(task))
			goto out_unlock_pi;
		/*
		 * If deadlock detection is off, we stop here if we
538 539 540
		 * are not the top pi waiter of the task. If deadlock
		 * detection is enabled we continue, but stop the
		 * requeueing in the chain walk.
541
		 */
542 543 544 545 546 547
		if (top_waiter != task_top_pi_waiter(task)) {
			if (!detect_deadlock)
				goto out_unlock_pi;
			else
				requeue = false;
		}
548
	}
I
Ingo Molnar 已提交
549 550

	/*
551 552 553 554 555
	 * If the waiter priority is the same as the task priority
	 * then there is no further priority adjustment necessary.  If
	 * deadlock detection is off, we stop the chain walk. If its
	 * enabled we continue, but stop the requeueing in the chain
	 * walk.
I
Ingo Molnar 已提交
556
	 */
557
	if (waiter->prio == task->prio && !dl_task(task)) {
558 559 560 561 562
		if (!detect_deadlock)
			goto out_unlock_pi;
		else
			requeue = false;
	}
I
Ingo Molnar 已提交
563

564 565 566
	/*
	 * [4] Get the next lock
	 */
I
Ingo Molnar 已提交
567
	lock = waiter->lock;
568 569 570 571 572
	/*
	 * [5] We need to trylock here as we are holding task->pi_lock,
	 * which is the reverse lock order versus the other rtmutex
	 * operations.
	 */
573
	if (!raw_spin_trylock(&lock->wait_lock)) {
574
		raw_spin_unlock_irq(&task->pi_lock);
I
Ingo Molnar 已提交
575 576 577 578
		cpu_relax();
		goto retry;
	}

579
	/*
580 581 582
	 * [6] check_exit_conditions_2() protected by task->pi_lock and
	 * lock->wait_lock.
	 *
583 584 585 586 587
	 * Deadlock detection. If the lock is the same as the original
	 * lock which caused us to walk the lock chain or if the
	 * current lock is owned by the task which initiated the chain
	 * walk, we detected a deadlock.
	 */
588
	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
589
		debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);
590
		raw_spin_unlock(&lock->wait_lock);
591
		ret = -EDEADLK;
I
Ingo Molnar 已提交
592 593 594
		goto out_unlock_pi;
	}

595 596 597 598 599 600 601 602 603 604
	/*
	 * If we just follow the lock chain for deadlock detection, no
	 * need to do all the requeue operations. To avoid a truckload
	 * of conditionals around the various places below, just do the
	 * minimum chain walk checks.
	 */
	if (!requeue) {
		/*
		 * No requeue[7] here. Just release @task [8]
		 */
605
		raw_spin_unlock(&task->pi_lock);
606 607 608 609 610 611 612
		put_task_struct(task);

		/*
		 * [9] check_exit_conditions_3 protected by lock->wait_lock.
		 * If there is no owner of the lock, end of chain.
		 */
		if (!rt_mutex_owner(lock)) {
613
			raw_spin_unlock_irq(&lock->wait_lock);
614 615 616 617 618 619
			return 0;
		}

		/* [10] Grab the next task, i.e. owner of @lock */
		task = rt_mutex_owner(lock);
		get_task_struct(task);
620
		raw_spin_lock(&task->pi_lock);
621 622 623 624 625 626 627 628 629 630 631 632 633 634

		/*
		 * No requeue [11] here. We just do deadlock detection.
		 *
		 * [12] Store whether owner is blocked
		 * itself. Decision is made after dropping the locks
		 */
		next_lock = task_blocked_on_lock(task);
		/*
		 * Get the top waiter for the next iteration
		 */
		top_waiter = rt_mutex_top_waiter(lock);

		/* [13] Drop locks */
635 636
		raw_spin_unlock(&task->pi_lock);
		raw_spin_unlock_irq(&lock->wait_lock);
637 638 639 640 641 642 643

		/* If owner is not blocked, end of chain. */
		if (!next_lock)
			goto out_put_task;
		goto again;
	}

644 645 646 647 648 649
	/*
	 * Store the current top waiter before doing the requeue
	 * operation on @lock. We need it for the boost/deboost
	 * decision below.
	 */
	prerequeue_top_waiter = rt_mutex_top_waiter(lock);
I
Ingo Molnar 已提交
650

651
	/* [7] Requeue the waiter in the lock waiter tree. */
652
	rt_mutex_dequeue(lock, waiter);
653
	waiter->prio = task->prio;
654
	rt_mutex_enqueue(lock, waiter);
I
Ingo Molnar 已提交
655

656
	/* [8] Release the task */
657
	raw_spin_unlock(&task->pi_lock);
658 659
	put_task_struct(task);

660
	/*
661 662
	 * [9] check_exit_conditions_3 protected by lock->wait_lock.
	 *
663 664 665 666
	 * We must abort the chain walk if there is no lock owner even
	 * in the dead lock detection case, as we have nothing to
	 * follow here. This is the end of the chain we are walking.
	 */
667 668
	if (!rt_mutex_owner(lock)) {
		/*
669 670 671
		 * If the requeue [7] above changed the top waiter,
		 * then we need to wake the new top waiter up to try
		 * to get the lock.
672
		 */
673
		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
674
			wake_up_process(rt_mutex_top_waiter(lock)->task);
675
		raw_spin_unlock_irq(&lock->wait_lock);
676
		return 0;
677
	}
I
Ingo Molnar 已提交
678

679
	/* [10] Grab the next task, i.e. the owner of @lock */
I
Ingo Molnar 已提交
680
	task = rt_mutex_owner(lock);
681
	get_task_struct(task);
682
	raw_spin_lock(&task->pi_lock);
I
Ingo Molnar 已提交
683

684
	/* [11] requeue the pi waiters if necessary */
I
Ingo Molnar 已提交
685
	if (waiter == rt_mutex_top_waiter(lock)) {
686 687 688
		/*
		 * The waiter became the new top (highest priority)
		 * waiter on the lock. Replace the previous top waiter
689
		 * in the owner tasks pi waiters tree with this waiter
690 691 692
		 * and adjust the priority of the owner.
		 */
		rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
693
		rt_mutex_enqueue_pi(task, waiter);
694
		rt_mutex_adjust_prio(task);
I
Ingo Molnar 已提交
695

696 697 698 699
	} else if (prerequeue_top_waiter == waiter) {
		/*
		 * The waiter was the top waiter on the lock, but is
		 * no longer the top prority waiter. Replace waiter in
700
		 * the owner tasks pi waiters tree with the new top
701 702 703 704 705 706
		 * (highest priority) waiter and adjust the priority
		 * of the owner.
		 * The new top waiter is stored in @waiter so that
		 * @waiter == @top_waiter evaluates to true below and
		 * we continue to deboost the rest of the chain.
		 */
707
		rt_mutex_dequeue_pi(task, waiter);
I
Ingo Molnar 已提交
708
		waiter = rt_mutex_top_waiter(lock);
709
		rt_mutex_enqueue_pi(task, waiter);
710
		rt_mutex_adjust_prio(task);
711 712 713 714 715
	} else {
		/*
		 * Nothing changed. No need to do any priority
		 * adjustment.
		 */
I
Ingo Molnar 已提交
716 717
	}

718
	/*
719 720 721 722
	 * [12] check_exit_conditions_4() protected by task->pi_lock
	 * and lock->wait_lock. The actual decisions are made after we
	 * dropped the locks.
	 *
723 724 725 726 727 728
	 * Check whether the task which owns the current lock is pi
	 * blocked itself. If yes we store a pointer to the lock for
	 * the lock chain change detection above. After we dropped
	 * task->pi_lock next_lock cannot be dereferenced anymore.
	 */
	next_lock = task_blocked_on_lock(task);
729 730 731 732
	/*
	 * Store the top waiter of @lock for the end of chain walk
	 * decision below.
	 */
I
Ingo Molnar 已提交
733
	top_waiter = rt_mutex_top_waiter(lock);
734 735

	/* [13] Drop the locks */
736 737
	raw_spin_unlock(&task->pi_lock);
	raw_spin_unlock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
738

739
	/*
740 741 742
	 * Make the actual exit decisions [12], based on the stored
	 * values.
	 *
743 744 745 746 747 748
	 * We reached the end of the lock chain. Stop right here. No
	 * point to go back just to figure that out.
	 */
	if (!next_lock)
		goto out_put_task;

749 750 751 752 753
	/*
	 * If the current waiter is not the top waiter on the lock,
	 * then we can stop the chain walk here if we are not in full
	 * deadlock detection mode.
	 */
I
Ingo Molnar 已提交
754 755 756 757 758 759
	if (!detect_deadlock && waiter != top_waiter)
		goto out_put_task;

	goto again;

 out_unlock_pi:
760
	raw_spin_unlock_irq(&task->pi_lock);
I
Ingo Molnar 已提交
761 762
 out_put_task:
	put_task_struct(task);
763

I
Ingo Molnar 已提交
764 765 766 767 768 769
	return ret;
}

/*
 * Try to take an rt-mutex
 *
770
 * Must be called with lock->wait_lock held and interrupts disabled
771
 *
772 773
 * @lock:   The lock to be acquired.
 * @task:   The task which wants to acquire the lock
774
 * @waiter: The waiter that is queued to the lock's wait tree if the
775
 *	    callsite called task_blocked_on_lock(), otherwise NULL
I
Ingo Molnar 已提交
776
 */
777
static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
778
				struct rt_mutex_waiter *waiter)
I
Ingo Molnar 已提交
779 780
{
	/*
781 782 783 784
	 * Before testing whether we can acquire @lock, we set the
	 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
	 * other tasks which try to modify @lock into the slow path
	 * and they serialize on @lock->wait_lock.
I
Ingo Molnar 已提交
785
	 *
786 787
	 * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
	 * as explained at the top of this file if and only if:
I
Ingo Molnar 已提交
788
	 *
789 790 791 792 793 794 795
	 * - There is a lock owner. The caller must fixup the
	 *   transient state if it does a trylock or leaves the lock
	 *   function due to a signal or timeout.
	 *
	 * - @task acquires the lock and there are no other
	 *   waiters. This is undone in rt_mutex_set_owner(@task) at
	 *   the end of this function.
I
Ingo Molnar 已提交
796 797 798
	 */
	mark_rt_mutex_waiters(lock);

799 800 801
	/*
	 * If @lock has an owner, give up.
	 */
802
	if (rt_mutex_owner(lock))
I
Ingo Molnar 已提交
803 804
		return 0;

805
	/*
806
	 * If @waiter != NULL, @task has already enqueued the waiter
807
	 * into @lock waiter tree. If @waiter == NULL then this is a
808
	 * trylock attempt.
809
	 */
810 811 812 813 814 815 816
	if (waiter) {
		/*
		 * If waiter is not the highest priority waiter of
		 * @lock, give up.
		 */
		if (waiter != rt_mutex_top_waiter(lock))
			return 0;
817

818 819
		/*
		 * We can acquire the lock. Remove the waiter from the
820
		 * lock waiters tree.
821 822
		 */
		rt_mutex_dequeue(lock, waiter);
823

824
	} else {
825
		/*
826 827 828 829 830 831
		 * If the lock has waiters already we check whether @task is
		 * eligible to take over the lock.
		 *
		 * If there are no other waiters, @task can acquire
		 * the lock.  @task->pi_blocked_on is NULL, so it does
		 * not need to be dequeued.
832 833
		 */
		if (rt_mutex_has_waiters(lock)) {
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
			/*
			 * If @task->prio is greater than or equal to
			 * the top waiter priority (kernel view),
			 * @task lost.
			 */
			if (task->prio >= rt_mutex_top_waiter(lock)->prio)
				return 0;

			/*
			 * The current top waiter stays enqueued. We
			 * don't have to change anything in the lock
			 * waiters order.
			 */
		} else {
			/*
			 * No waiters. Take the lock without the
			 * pi_lock dance.@task->pi_blocked_on is NULL
			 * and we have no waiters to enqueue in @task
852
			 * pi waiters tree.
853 854
			 */
			goto takeit;
855 856 857
		}
	}

858 859 860 861 862 863
	/*
	 * Clear @task->pi_blocked_on. Requires protection by
	 * @task->pi_lock. Redundant operation for the @waiter == NULL
	 * case, but conditionals are more expensive than a redundant
	 * store.
	 */
864
	raw_spin_lock(&task->pi_lock);
865 866 867 868
	task->pi_blocked_on = NULL;
	/*
	 * Finish the lock acquisition. @task is the new owner. If
	 * other waiters exist we have to insert the highest priority
869
	 * waiter into @task->pi_waiters tree.
870 871 872
	 */
	if (rt_mutex_has_waiters(lock))
		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
873
	raw_spin_unlock(&task->pi_lock);
874 875

takeit:
I
Ingo Molnar 已提交
876
	/* We got the lock. */
877
	debug_rt_mutex_lock(lock);
I
Ingo Molnar 已提交
878

879 880 881 882
	/*
	 * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
	 * are still waiters or clears it.
	 */
883
	rt_mutex_set_owner(lock, task);
I
Ingo Molnar 已提交
884 885 886 887 888 889 890 891 892

	return 1;
}

/*
 * Task blocks on lock.
 *
 * Prepare waiter and propagate pi chain
 *
893
 * This must be called with lock->wait_lock held and interrupts disabled
I
Ingo Molnar 已提交
894 895 896
 */
static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
				   struct rt_mutex_waiter *waiter,
D
Darren Hart 已提交
897
				   struct task_struct *task,
898
				   enum rtmutex_chainwalk chwalk)
I
Ingo Molnar 已提交
899
{
900
	struct task_struct *owner = rt_mutex_owner(lock);
I
Ingo Molnar 已提交
901
	struct rt_mutex_waiter *top_waiter = waiter;
902
	struct rt_mutex *next_lock;
903
	int chain_walk = 0, res;
I
Ingo Molnar 已提交
904

905 906 907 908 909 910 911 912 913
	/*
	 * Early deadlock detection. We really don't want the task to
	 * enqueue on itself just to untangle the mess later. It's not
	 * only an optimization. We drop the locks, so another waiter
	 * can come in before the chain walk detects the deadlock. So
	 * the other will detect the deadlock and return -EDEADLOCK,
	 * which is wrong, as the other waiter is not in a deadlock
	 * situation.
	 */
914
	if (owner == task)
915 916
		return -EDEADLK;

917
	raw_spin_lock(&task->pi_lock);
918
	rt_mutex_adjust_prio(task);
D
Darren Hart 已提交
919
	waiter->task = task;
I
Ingo Molnar 已提交
920
	waiter->lock = lock;
921
	waiter->prio = task->prio;
I
Ingo Molnar 已提交
922 923 924 925

	/* Get the top priority waiter on the lock */
	if (rt_mutex_has_waiters(lock))
		top_waiter = rt_mutex_top_waiter(lock);
926
	rt_mutex_enqueue(lock, waiter);
I
Ingo Molnar 已提交
927

D
Darren Hart 已提交
928
	task->pi_blocked_on = waiter;
I
Ingo Molnar 已提交
929

930
	raw_spin_unlock(&task->pi_lock);
I
Ingo Molnar 已提交
931

932 933 934
	if (!owner)
		return 0;

935
	raw_spin_lock(&owner->pi_lock);
I
Ingo Molnar 已提交
936
	if (waiter == rt_mutex_top_waiter(lock)) {
937 938
		rt_mutex_dequeue_pi(owner, top_waiter);
		rt_mutex_enqueue_pi(owner, waiter);
I
Ingo Molnar 已提交
939

940
		rt_mutex_adjust_prio(owner);
941 942
		if (owner->pi_blocked_on)
			chain_walk = 1;
943
	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
944
		chain_walk = 1;
945
	}
946

947 948 949
	/* Store the lock on which owner is blocked or NULL */
	next_lock = task_blocked_on_lock(owner);

950
	raw_spin_unlock(&owner->pi_lock);
951 952 953 954 955 956
	/*
	 * Even if full deadlock detection is on, if the owner is not
	 * blocked itself, we can avoid finding this out in the chain
	 * walk.
	 */
	if (!chain_walk || !next_lock)
I
Ingo Molnar 已提交
957 958
		return 0;

959 960 961 962 963 964 965
	/*
	 * The owner can't disappear while holding a lock,
	 * so the owner struct is protected by wait_lock.
	 * Gets dropped in rt_mutex_adjust_prio_chain()!
	 */
	get_task_struct(owner);

966
	raw_spin_unlock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
967

968
	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
969
					 next_lock, waiter, task);
I
Ingo Molnar 已提交
970

971
	raw_spin_lock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
972 973 974 975 976

	return res;
}

/*
977
 * Remove the top waiter from the current tasks pi waiter tree and
978
 * queue it up.
I
Ingo Molnar 已提交
979
 *
980
 * Called with lock->wait_lock held and interrupts disabled.
I
Ingo Molnar 已提交
981
 */
982 983
static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
				    struct rt_mutex *lock)
I
Ingo Molnar 已提交
984 985 986
{
	struct rt_mutex_waiter *waiter;

987
	raw_spin_lock(&current->pi_lock);
I
Ingo Molnar 已提交
988 989 990 991

	waiter = rt_mutex_top_waiter(lock);

	/*
992 993 994 995 996
	 * Remove it from current->pi_waiters and deboost.
	 *
	 * We must in fact deboost here in order to ensure we call
	 * rt_mutex_setprio() to update p->pi_top_task before the
	 * task unblocks.
I
Ingo Molnar 已提交
997
	 */
998
	rt_mutex_dequeue_pi(current, waiter);
999
	rt_mutex_adjust_prio(current);
I
Ingo Molnar 已提交
1000

T
Thomas Gleixner 已提交
1001 1002 1003 1004 1005 1006 1007 1008 1009
	/*
	 * As we are waking up the top waiter, and the waiter stays
	 * queued on the lock until it gets the lock, this lock
	 * obviously has waiters. Just set the bit here and this has
	 * the added benefit of forcing all new tasks into the
	 * slow path making sure no task of lower priority than
	 * the top waiter can steal this lock.
	 */
	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
I
Ingo Molnar 已提交
1010

1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	/*
	 * We deboosted before waking the top waiter task such that we don't
	 * run two tasks with the 'same' priority (and ensure the
	 * p->pi_top_task pointer points to a blocked task). This however can
	 * lead to priority inversion if we would get preempted after the
	 * deboost but before waking our donor task, hence the preempt_disable()
	 * before unlock.
	 *
	 * Pairs with preempt_enable() in rt_mutex_postunlock();
	 */
	preempt_disable();
1022
	wake_q_add(wake_q, waiter->task);
1023
	raw_spin_unlock(&current->pi_lock);
I
Ingo Molnar 已提交
1024 1025 1026
}

/*
1027
 * Remove a waiter from a lock and give up
I
Ingo Molnar 已提交
1028
 *
1029
 * Must be called with lock->wait_lock held and interrupts disabled. I must
1030
 * have just failed to try_to_take_rt_mutex().
I
Ingo Molnar 已提交
1031
 */
1032 1033
static void remove_waiter(struct rt_mutex *lock,
			  struct rt_mutex_waiter *waiter)
I
Ingo Molnar 已提交
1034
{
1035
	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
1036
	struct task_struct *owner = rt_mutex_owner(lock);
1037
	struct rt_mutex *next_lock;
I
Ingo Molnar 已提交
1038

1039
	raw_spin_lock(&current->pi_lock);
1040
	rt_mutex_dequeue(lock, waiter);
I
Ingo Molnar 已提交
1041
	current->pi_blocked_on = NULL;
1042
	raw_spin_unlock(&current->pi_lock);
I
Ingo Molnar 已提交
1043

1044 1045 1046 1047 1048
	/*
	 * Only update priority if the waiter was the highest priority
	 * waiter of the lock and there is an owner to update.
	 */
	if (!owner || !is_top_waiter)
1049 1050
		return;

1051
	raw_spin_lock(&owner->pi_lock);
I
Ingo Molnar 已提交
1052

1053
	rt_mutex_dequeue_pi(owner, waiter);
I
Ingo Molnar 已提交
1054

1055 1056
	if (rt_mutex_has_waiters(lock))
		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
I
Ingo Molnar 已提交
1057

1058
	rt_mutex_adjust_prio(owner);
I
Ingo Molnar 已提交
1059

1060 1061
	/* Store the lock on which owner is blocked or NULL */
	next_lock = task_blocked_on_lock(owner);
1062

1063
	raw_spin_unlock(&owner->pi_lock);
I
Ingo Molnar 已提交
1064

1065 1066 1067 1068
	/*
	 * Don't walk the chain, if the owner task is not blocked
	 * itself.
	 */
1069
	if (!next_lock)
I
Ingo Molnar 已提交
1070 1071
		return;

1072 1073 1074
	/* gets dropped in rt_mutex_adjust_prio_chain()! */
	get_task_struct(owner);

1075
	raw_spin_unlock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
1076

1077 1078
	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
				   next_lock, NULL, current);
I
Ingo Molnar 已提交
1079

1080
	raw_spin_lock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
1081 1082
}

1083 1084 1085 1086 1087 1088 1089 1090
/*
 * Recheck the pi chain, in case we got a priority setting
 *
 * Called from sched_setscheduler
 */
void rt_mutex_adjust_pi(struct task_struct *task)
{
	struct rt_mutex_waiter *waiter;
1091
	struct rt_mutex *next_lock;
1092 1093
	unsigned long flags;

1094
	raw_spin_lock_irqsave(&task->pi_lock, flags);
1095 1096

	waiter = task->pi_blocked_on;
1097
	if (!waiter || (waiter->prio == task->prio && !dl_prio(task->prio))) {
1098
		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
1099 1100
		return;
	}
1101
	next_lock = waiter->lock;
1102
	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
1103

1104 1105
	/* gets dropped in rt_mutex_adjust_prio_chain()! */
	get_task_struct(task);
1106

1107 1108
	rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
				   next_lock, NULL, task);
1109 1110
}

1111 1112 1113 1114 1115 1116 1117 1118
void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
{
	debug_rt_mutex_init_waiter(waiter);
	RB_CLEAR_NODE(&waiter->pi_tree_entry);
	RB_CLEAR_NODE(&waiter->tree_entry);
	waiter->task = NULL;
}

D
Darren Hart 已提交
1119 1120 1121 1122
/**
 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
 * @lock:		 the rt_mutex to take
 * @state:		 the state the task should block in (TASK_INTERRUPTIBLE
1123
 *			 or TASK_UNINTERRUPTIBLE)
D
Darren Hart 已提交
1124 1125 1126
 * @timeout:		 the pre-initialized and started timer, or NULL for none
 * @waiter:		 the pre-initialized rt_mutex_waiter
 *
1127
 * Must be called with lock->wait_lock held and interrupts disabled
I
Ingo Molnar 已提交
1128 1129
 */
static int __sched
D
Darren Hart 已提交
1130 1131
__rt_mutex_slowlock(struct rt_mutex *lock, int state,
		    struct hrtimer_sleeper *timeout,
1132
		    struct rt_mutex_waiter *waiter)
I
Ingo Molnar 已提交
1133 1134 1135 1136 1137
{
	int ret = 0;

	for (;;) {
		/* Try to acquire the lock: */
1138
		if (try_to_take_rt_mutex(lock, current, waiter))
I
Ingo Molnar 已提交
1139 1140 1141 1142 1143 1144
			break;

		/*
		 * TASK_INTERRUPTIBLE checks for signals and
		 * timeout. Ignored otherwise.
		 */
1145
		if (likely(state == TASK_INTERRUPTIBLE)) {
I
Ingo Molnar 已提交
1146 1147 1148 1149 1150 1151 1152 1153 1154
			/* Signal pending? */
			if (signal_pending(current))
				ret = -EINTR;
			if (timeout && !timeout->task)
				ret = -ETIMEDOUT;
			if (ret)
				break;
		}

1155
		raw_spin_unlock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
1156

D
Darren Hart 已提交
1157
		debug_rt_mutex_print_deadlock(waiter);
I
Ingo Molnar 已提交
1158

1159
		schedule();
I
Ingo Molnar 已提交
1160

1161
		raw_spin_lock_irq(&lock->wait_lock);
I
Ingo Molnar 已提交
1162 1163 1164
		set_current_state(state);
	}

1165
	__set_current_state(TASK_RUNNING);
D
Darren Hart 已提交
1166 1167 1168
	return ret;
}

1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
				     struct rt_mutex_waiter *w)
{
	/*
	 * If the result is not -EDEADLOCK or the caller requested
	 * deadlock detection, nothing to do here.
	 */
	if (res != -EDEADLOCK || detect_deadlock)
		return;

	/*
	 * Yell lowdly and stop the task right here.
	 */
	rt_mutex_print_deadlock(w);
	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);
		schedule();
	}
}

D
Darren Hart 已提交
1189 1190 1191 1192 1193 1194
/*
 * Slow path lock function:
 */
static int __sched
rt_mutex_slowlock(struct rt_mutex *lock, int state,
		  struct hrtimer_sleeper *timeout,
1195
		  enum rtmutex_chainwalk chwalk)
D
Darren Hart 已提交
1196 1197
{
	struct rt_mutex_waiter waiter;
1198
	unsigned long flags;
D
Darren Hart 已提交
1199 1200
	int ret = 0;

1201
	rt_mutex_init_waiter(&waiter);
D
Darren Hart 已提交
1202

1203 1204 1205 1206 1207 1208 1209 1210 1211
	/*
	 * Technically we could use raw_spin_[un]lock_irq() here, but this can
	 * be called in early boot if the cmpxchg() fast path is disabled
	 * (debug, no architecture support). In this case we will acquire the
	 * rtmutex with lock->wait_lock held. But we cannot unconditionally
	 * enable interrupts in that early boot case. So we need to use the
	 * irqsave/restore variants.
	 */
	raw_spin_lock_irqsave(&lock->wait_lock, flags);
D
Darren Hart 已提交
1212 1213

	/* Try to acquire the lock again: */
1214
	if (try_to_take_rt_mutex(lock, current, NULL)) {
1215
		raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
D
Darren Hart 已提交
1216 1217 1218 1219 1220 1221
		return 0;
	}

	set_current_state(state);

	/* Setup the timer, when timeout != NULL */
1222
	if (unlikely(timeout))
D
Darren Hart 已提交
1223 1224
		hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);

1225
	ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
1226 1227

	if (likely(!ret))
1228
		/* sleep on the mutex */
1229
		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
D
Darren Hart 已提交
1230

1231
	if (unlikely(ret)) {
1232
		__set_current_state(TASK_RUNNING);
1233 1234
		if (rt_mutex_has_waiters(lock))
			remove_waiter(lock, &waiter);
1235
		rt_mutex_handle_deadlock(ret, chwalk, &waiter);
1236
	}
I
Ingo Molnar 已提交
1237 1238 1239 1240 1241 1242 1243

	/*
	 * try_to_take_rt_mutex() sets the waiter bit
	 * unconditionally. We might have to fix that up.
	 */
	fixup_rt_mutex_waiters(lock);

1244
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257

	/* Remove pending timer: */
	if (unlikely(timeout))
		hrtimer_cancel(&timeout->timer);

	debug_rt_mutex_free_waiter(&waiter);

	return ret;
}

/*
 * Slow path try-lock function:
 */
1258
static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
I
Ingo Molnar 已提交
1259
{
1260
	unsigned long flags;
1261 1262 1263 1264 1265 1266 1267 1268 1269
	int ret;

	/*
	 * If the lock already has an owner we fail to get the lock.
	 * This can be done without taking the @lock->wait_lock as
	 * it is only being read, and this is a trylock anyway.
	 */
	if (rt_mutex_owner(lock))
		return 0;
I
Ingo Molnar 已提交
1270

1271
	/*
1272 1273
	 * The mutex has currently no owner. Lock the wait lock and try to
	 * acquire the lock. We use irqsave here to support early boot calls.
1274
	 */
1275
	raw_spin_lock_irqsave(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1276

1277
	ret = try_to_take_rt_mutex(lock, current, NULL);
I
Ingo Molnar 已提交
1278

1279 1280 1281 1282 1283
	/*
	 * try_to_take_rt_mutex() sets the lock waiters bit
	 * unconditionally. Clean this up.
	 */
	fixup_rt_mutex_waiters(lock);
I
Ingo Molnar 已提交
1284

1285
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1286 1287 1288 1289 1290

	return ret;
}

/*
1291
 * Slow path to release a rt-mutex.
P
Peter Zijlstra 已提交
1292 1293
 *
 * Return whether the current task needs to call rt_mutex_postunlock().
I
Ingo Molnar 已提交
1294
 */
1295 1296
static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
					struct wake_q_head *wake_q)
I
Ingo Molnar 已提交
1297
{
1298 1299 1300 1301
	unsigned long flags;

	/* irqsave required to support early boot calls */
	raw_spin_lock_irqsave(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1302 1303 1304

	debug_rt_mutex_unlock(lock);

T
Thomas Gleixner 已提交
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
	/*
	 * We must be careful here if the fast path is enabled. If we
	 * have no waiters queued we cannot set owner to NULL here
	 * because of:
	 *
	 * foo->lock->owner = NULL;
	 *			rtmutex_lock(foo->lock);   <- fast path
	 *			free = atomic_dec_and_test(foo->refcnt);
	 *			rtmutex_unlock(foo->lock); <- fast path
	 *			if (free)
	 *				kfree(foo);
	 * raw_spin_unlock(foo->lock->wait_lock);
	 *
	 * So for the fastpath enabled kernel:
	 *
	 * Nothing can set the waiters bit as long as we hold
	 * lock->wait_lock. So we do the following sequence:
	 *
	 *	owner = rt_mutex_owner(lock);
	 *	clear_rt_mutex_waiters(lock);
	 *	raw_spin_unlock(&lock->wait_lock);
	 *	if (cmpxchg(&lock->owner, owner, 0) == owner)
	 *		return;
	 *	goto retry;
	 *
	 * The fastpath disabled variant is simple as all access to
	 * lock->owner is serialized by lock->wait_lock:
	 *
	 *	lock->owner = NULL;
	 *	raw_spin_unlock(&lock->wait_lock);
	 */
	while (!rt_mutex_has_waiters(lock)) {
		/* Drops lock->wait_lock ! */
1338
		if (unlock_rt_mutex_safe(lock, flags) == true)
1339
			return false;
T
Thomas Gleixner 已提交
1340
		/* Relock the rtmutex and try again */
1341
		raw_spin_lock_irqsave(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1342 1343
	}

T
Thomas Gleixner 已提交
1344 1345 1346
	/*
	 * The wakeup next waiter path does not suffer from the above
	 * race. See the comments there.
1347 1348
	 *
	 * Queue the next waiter for wakeup once we release the wait_lock.
T
Thomas Gleixner 已提交
1349
	 */
1350
	mark_wakeup_next_waiter(wake_q, lock);
1351
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
I
Ingo Molnar 已提交
1352

P
Peter Zijlstra 已提交
1353
	return true; /* call rt_mutex_postunlock() */
I
Ingo Molnar 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
}

/*
 * debug aware fast / slowpath lock,trylock,unlock
 *
 * The atomic acquire/release ops are compiled away, when either the
 * architecture does not support cmpxchg or when debugging is enabled.
 */
static inline int
rt_mutex_fastlock(struct rt_mutex *lock, int state,
		  int (*slowfn)(struct rt_mutex *lock, int state,
				struct hrtimer_sleeper *timeout,
1366
				enum rtmutex_chainwalk chwalk))
I
Ingo Molnar 已提交
1367
{
1368
	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
I
Ingo Molnar 已提交
1369
		return 0;
1370 1371

	return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
I
Ingo Molnar 已提交
1372 1373 1374 1375
}

static inline int
rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
1376 1377
			struct hrtimer_sleeper *timeout,
			enum rtmutex_chainwalk chwalk,
I
Ingo Molnar 已提交
1378 1379
			int (*slowfn)(struct rt_mutex *lock, int state,
				      struct hrtimer_sleeper *timeout,
1380
				      enum rtmutex_chainwalk chwalk))
I
Ingo Molnar 已提交
1381
{
1382
	if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
1383
	    likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
I
Ingo Molnar 已提交
1384
		return 0;
1385 1386

	return slowfn(lock, state, timeout, chwalk);
I
Ingo Molnar 已提交
1387 1388 1389 1390
}

static inline int
rt_mutex_fasttrylock(struct rt_mutex *lock,
1391
		     int (*slowfn)(struct rt_mutex *lock))
I
Ingo Molnar 已提交
1392
{
1393
	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
I
Ingo Molnar 已提交
1394
		return 1;
1395

1396
	return slowfn(lock);
I
Ingo Molnar 已提交
1397 1398
}

1399
/*
P
Peter Zijlstra 已提交
1400
 * Performs the wakeup of the the top-waiter and re-enables preemption.
1401
 */
P
Peter Zijlstra 已提交
1402
void rt_mutex_postunlock(struct wake_q_head *wake_q)
1403 1404 1405 1406
{
	wake_up_q(wake_q);

	/* Pairs with preempt_disable() in rt_mutex_slowunlock() */
P
Peter Zijlstra 已提交
1407
	preempt_enable();
1408 1409
}

I
Ingo Molnar 已提交
1410 1411
static inline void
rt_mutex_fastunlock(struct rt_mutex *lock,
1412 1413
		    bool (*slowfn)(struct rt_mutex *lock,
				   struct wake_q_head *wqh))
I
Ingo Molnar 已提交
1414
{
1415
	DEFINE_WAKE_Q(wake_q);
1416

1417 1418
	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
		return;
1419

P
Peter Zijlstra 已提交
1420 1421
	if (slowfn(lock, &wake_q))
		rt_mutex_postunlock(&wake_q);
I
Ingo Molnar 已提交
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
}

/**
 * rt_mutex_lock - lock a rt_mutex
 *
 * @lock: the rt_mutex to be locked
 */
void __sched rt_mutex_lock(struct rt_mutex *lock)
{
	might_sleep();

1433
	rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
I
Ingo Molnar 已提交
1434 1435 1436 1437 1438 1439
}
EXPORT_SYMBOL_GPL(rt_mutex_lock);

/**
 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
 *
1440
 * @lock:		the rt_mutex to be locked
I
Ingo Molnar 已提交
1441 1442
 *
 * Returns:
1443 1444
 *  0		on success
 * -EINTR	when interrupted by a signal
I
Ingo Molnar 已提交
1445
 */
1446
int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
I
Ingo Molnar 已提交
1447 1448 1449
{
	might_sleep();

1450
	return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
I
Ingo Molnar 已提交
1451 1452 1453
}
EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);

1454 1455 1456 1457 1458 1459
/*
 * Futex variant, must not use fastpath.
 */
int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
{
	return rt_mutex_slowtrylock(lock);
1460 1461
}

I
Ingo Molnar 已提交
1462
/**
1463 1464 1465
 * rt_mutex_timed_lock - lock a rt_mutex interruptible
 *			the timeout structure is provided
 *			by the caller
I
Ingo Molnar 已提交
1466
 *
1467
 * @lock:		the rt_mutex to be locked
I
Ingo Molnar 已提交
1468 1469 1470
 * @timeout:		timeout structure or NULL (no timeout)
 *
 * Returns:
1471 1472
 *  0		on success
 * -EINTR	when interrupted by a signal
1473
 * -ETIMEDOUT	when the timeout expired
I
Ingo Molnar 已提交
1474 1475
 */
int
1476
rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
I
Ingo Molnar 已提交
1477 1478 1479
{
	might_sleep();

1480 1481
	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
				       RT_MUTEX_MIN_CHAINWALK,
1482
				       rt_mutex_slowlock);
I
Ingo Molnar 已提交
1483 1484 1485 1486 1487 1488 1489 1490
}
EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);

/**
 * rt_mutex_trylock - try to lock a rt_mutex
 *
 * @lock:	the rt_mutex to be locked
 *
1491 1492 1493 1494
 * This function can only be called in thread context. It's safe to
 * call it from atomic regions, but not from hard interrupt or soft
 * interrupt context.
 *
I
Ingo Molnar 已提交
1495 1496 1497 1498
 * Returns 1 on success and 0 on contention
 */
int __sched rt_mutex_trylock(struct rt_mutex *lock)
{
1499
	if (WARN_ON_ONCE(in_irq() || in_nmi() || in_serving_softirq()))
1500 1501
		return 0;

I
Ingo Molnar 已提交
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
	return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
}
EXPORT_SYMBOL_GPL(rt_mutex_trylock);

/**
 * rt_mutex_unlock - unlock a rt_mutex
 *
 * @lock: the rt_mutex to be unlocked
 */
void __sched rt_mutex_unlock(struct rt_mutex *lock)
{
	rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_unlock);

1517
/**
1518 1519
 * Futex variant, that since futex variants do not use the fast-path, can be
 * simple and will not need to retry.
1520
 */
1521 1522
bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
				    struct wake_q_head *wake_q)
1523
{
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
	lockdep_assert_held(&lock->wait_lock);

	debug_rt_mutex_unlock(lock);

	if (!rt_mutex_has_waiters(lock)) {
		lock->owner = NULL;
		return false; /* done */
	}

	mark_wakeup_next_waiter(wake_q, lock);
1534 1535 1536 1537 1538 1539 1540
	/*
	 * We've already deboosted, retain preempt_disabled when dropping
	 * the wait_lock to avoid inversion until the wakeup. Matched
	 * by rt_mutex_postunlock();
	 */
	preempt_disable();

P
Peter Zijlstra 已提交
1541
	return true; /* call postunlock() */
1542
}
1543

1544 1545 1546
void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
{
	DEFINE_WAKE_Q(wake_q);
P
Peter Zijlstra 已提交
1547
	bool postunlock;
1548 1549

	raw_spin_lock_irq(&lock->wait_lock);
P
Peter Zijlstra 已提交
1550
	postunlock = __rt_mutex_futex_unlock(lock, &wake_q);
1551 1552
	raw_spin_unlock_irq(&lock->wait_lock);

P
Peter Zijlstra 已提交
1553 1554
	if (postunlock)
		rt_mutex_postunlock(&wake_q);
1555 1556
}

1557
/**
I
Ingo Molnar 已提交
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
 * rt_mutex_destroy - mark a mutex unusable
 * @lock: the mutex to be destroyed
 *
 * This function marks the mutex uninitialized, and any subsequent
 * use of the mutex is forbidden. The mutex must not be locked when
 * this function is called.
 */
void rt_mutex_destroy(struct rt_mutex *lock)
{
	WARN_ON(rt_mutex_is_locked(lock));
#ifdef CONFIG_DEBUG_RT_MUTEXES
	lock->magic = NULL;
#endif
}

EXPORT_SYMBOL_GPL(rt_mutex_destroy);

/**
 * __rt_mutex_init - initialize the rt lock
 *
 * @lock: the rt lock to be initialized
 *
 * Initialize the rt lock to unlocked state.
 *
 * Initializing of a locked rt lock is not allowed
 */
void __rt_mutex_init(struct rt_mutex *lock, const char *name)
{
	lock->owner = NULL;
1587
	raw_spin_lock_init(&lock->wait_lock);
1588 1589
	lock->waiters = RB_ROOT;
	lock->waiters_leftmost = NULL;
I
Ingo Molnar 已提交
1590 1591 1592 1593

	debug_rt_mutex_init(lock, name);
}
EXPORT_SYMBOL_GPL(__rt_mutex_init);
1594 1595 1596 1597 1598

/**
 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
 *				proxy owner
 *
1599
 * @lock:	the rt_mutex to be locked
1600 1601 1602
 * @proxy_owner:the task to set as owner
 *
 * No locking. Caller has to do serializing itself
1603 1604 1605 1606 1607
 *
 * Special API call for PI-futex support. This initializes the rtmutex and
 * assigns it to @proxy_owner. Concurrent operations on the rtmutex are not
 * possible at this point because the pi_state which contains the rtmutex
 * is not yet visible to other tasks.
1608 1609 1610 1611 1612
 */
void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
				struct task_struct *proxy_owner)
{
	__rt_mutex_init(lock, NULL);
1613
	debug_rt_mutex_proxy_lock(lock, proxy_owner);
1614
	rt_mutex_set_owner(lock, proxy_owner);
1615 1616 1617 1618 1619
}

/**
 * rt_mutex_proxy_unlock - release a lock on behalf of owner
 *
1620
 * @lock:	the rt_mutex to be locked
1621 1622
 *
 * No locking. Caller has to do serializing itself
1623 1624 1625 1626 1627
 *
 * Special API call for PI-futex support. This merrily cleans up the rtmutex
 * (debugging) state. Concurrent operations on this rt_mutex are not
 * possible because it belongs to the pi_state which is about to be freed
 * and it is not longer visible to other tasks.
1628 1629 1630 1631 1632
 */
void rt_mutex_proxy_unlock(struct rt_mutex *lock,
			   struct task_struct *proxy_owner)
{
	debug_rt_mutex_proxy_unlock(lock);
1633
	rt_mutex_set_owner(lock, NULL);
1634 1635
}

1636
int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
D
Darren Hart 已提交
1637
			      struct rt_mutex_waiter *waiter,
1638
			      struct task_struct *task)
D
Darren Hart 已提交
1639 1640 1641
{
	int ret;

1642
	if (try_to_take_rt_mutex(lock, task, NULL))
D
Darren Hart 已提交
1643 1644
		return 1;

1645
	/* We enforce deadlock detection for futexes */
1646 1647
	ret = task_blocks_on_rt_mutex(lock, waiter, task,
				      RT_MUTEX_FULL_CHAINWALK);
D
Darren Hart 已提交
1648

1649
	if (ret && !rt_mutex_owner(lock)) {
D
Darren Hart 已提交
1650 1651 1652 1653 1654 1655 1656 1657
		/*
		 * Reset the return value. We might have
		 * returned with -EDEADLK and the owner
		 * released the lock while we were walking the
		 * pi chain.  Let the waiter sort it out.
		 */
		ret = 0;
	}
1658 1659 1660 1661

	if (unlikely(ret))
		remove_waiter(lock, waiter);

D
Darren Hart 已提交
1662 1663 1664 1665 1666
	debug_rt_mutex_print_deadlock(waiter);

	return ret;
}

1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
/**
 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
 * @lock:		the rt_mutex to take
 * @waiter:		the pre-initialized rt_mutex_waiter
 * @task:		the task to prepare
 *
 * Returns:
 *  0 - task blocked on lock
 *  1 - acquired the lock for task, caller should wake it up
 * <0 - error
 *
 * Special API call for FUTEX_REQUEUE_PI support.
 */
int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
			      struct rt_mutex_waiter *waiter,
			      struct task_struct *task)
{
	int ret;

	raw_spin_lock_irq(&lock->wait_lock);
	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);
	raw_spin_unlock_irq(&lock->wait_lock);

	return ret;
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
/**
 * rt_mutex_next_owner - return the next owner of the lock
 *
 * @lock: the rt lock query
 *
 * Returns the next owner of the lock or NULL
 *
 * Caller has to serialize against other accessors to the lock
 * itself.
 *
 * Special API call for PI-futex support
 */
struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
{
	if (!rt_mutex_has_waiters(lock))
		return NULL;

	return rt_mutex_top_waiter(lock)->task;
}
D
Darren Hart 已提交
1712 1713

/**
1714
 * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
D
Darren Hart 已提交
1715 1716
 * @lock:		the rt_mutex we were woken on
 * @to:			the timeout, null if none. hrtimer should already have
1717
 *			been started.
D
Darren Hart 已提交
1718 1719
 * @waiter:		the pre-initialized rt_mutex_waiter
 *
1720 1721 1722
 * Wait for the the lock acquisition started on our behalf by
 * rt_mutex_start_proxy_lock(). Upon failure, the caller must call
 * rt_mutex_cleanup_proxy_lock().
D
Darren Hart 已提交
1723 1724 1725
 *
 * Returns:
 *  0 - success
1726
 * <0 - error, one of -EINTR, -ETIMEDOUT
D
Darren Hart 已提交
1727
 *
1728
 * Special API call for PI-futex support
D
Darren Hart 已提交
1729
 */
1730
int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
D
Darren Hart 已提交
1731
			       struct hrtimer_sleeper *to,
1732
			       struct rt_mutex_waiter *waiter)
D
Darren Hart 已提交
1733 1734 1735
{
	int ret;

1736
	raw_spin_lock_irq(&lock->wait_lock);
D
Darren Hart 已提交
1737 1738 1739

	set_current_state(TASK_INTERRUPTIBLE);

1740
	/* sleep on the mutex */
1741
	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
D
Darren Hart 已提交
1742

1743
	raw_spin_unlock_irq(&lock->wait_lock);
D
Darren Hart 已提交
1744 1745 1746

	return ret;
}
1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781

/**
 * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
 * @lock:		the rt_mutex we were woken on
 * @waiter:		the pre-initialized rt_mutex_waiter
 *
 * Attempt to clean up after a failed rt_mutex_wait_proxy_lock().
 *
 * Unless we acquired the lock; we're still enqueued on the wait-list and can
 * in fact still be granted ownership until we're removed. Therefore we can
 * find we are in fact the owner and must disregard the
 * rt_mutex_wait_proxy_lock() failure.
 *
 * Returns:
 *  true  - did the cleanup, we done.
 *  false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
 *          caller should disregards its return value.
 *
 * Special API call for PI-futex support
 */
bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
				 struct rt_mutex_waiter *waiter)
{
	bool cleanup = false;

	raw_spin_lock_irq(&lock->wait_lock);
	/*
	 * Unless we're the owner; we're still enqueued on the wait_list.
	 * So check if we became owner, if not, take us off the wait_list.
	 */
	if (rt_mutex_owner(lock) != current) {
		remove_waiter(lock, waiter);
		fixup_rt_mutex_waiters(lock);
		cleanup = true;
	}
1782 1783 1784 1785 1786 1787 1788

	/*
	 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
	 * have to fix that up.
	 */
	fixup_rt_mutex_waiters(lock);

1789 1790 1791 1792
	raw_spin_unlock_irq(&lock->wait_lock);

	return cleanup;
}