qspinlock.c 15.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
 * Queued spinlock
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
 * (C) Copyright 2013-2014 Red Hat, Inc.
 * (C) Copyright 2015 Intel Corp.
17
 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
18
 *
19
 * Authors: Waiman Long <waiman.long@hpe.com>
20
21
 *          Peter Zijlstra <peterz@infradead.org>
 */
22
23
24

#ifndef _GEN_PV_LOCK_SLOWPATH

25
26
27
28
29
30
#include <linux/smp.h>
#include <linux/bug.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
31
#include <asm/byteorder.h>
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#include <asm/qspinlock.h>

/*
 * The basic principle of a queue-based spinlock can best be understood
 * by studying a classic queue-based spinlock implementation called the
 * MCS lock. The paper below provides a good description for this kind
 * of lock.
 *
 * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
 *
 * This queued spinlock implementation is based on the MCS lock, however to make
 * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
 * API, we must modify it somehow.
 *
 * In particular; where the traditional MCS lock consists of a tail pointer
 * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
 * unlock the next pending (next->locked), we compress both these: {tail,
 * next->locked} into a single u32 value.
 *
 * Since a spinlock disables recursion of its own context and there is a limit
 * to the contexts that can nest; namely: task, softirq, hardirq, nmi. As there
 * are at most 4 nesting levels, it can be encoded by a 2-bit number. Now
 * we can encode the tail by combining the 2-bit nesting level with the cpu
 * number. With one byte for the lock value and 3 bytes for the tail, only a
 * 32-bit word is now needed. Even though we only need 1 bit for the lock,
 * we extend it to a full byte to achieve better performance for architectures
 * that support atomic byte write.
 *
 * We also change the first spinner to spin on the lock bit instead of its
 * node; whereby avoiding the need to carry a node from lock to unlock, and
 * preserving existing lock API. This also makes the unlock code simpler and
 * faster.
64
65
66
67
 *
 * N.B. The current implementation only supports architectures that allow
 *      atomic operations on smaller 8-bit and 16-bit data types.
 *
68
69
70
71
 */

#include "mcs_spinlock.h"

72
73
74
75
76
77
#ifdef CONFIG_PARAVIRT_SPINLOCKS
#define MAX_NODES	8
#else
#define MAX_NODES	4
#endif

78
79
80
81
82
/*
 * Per-CPU queue node structures; we can never have more than 4 nested
 * contexts: task, softirq, hardirq, nmi.
 *
 * Exactly fits one 64-byte cacheline on a 64-bit architecture.
83
84
 *
 * PV doubles the storage and uses the second cacheline for PV state.
85
 */
86
static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

/*
 * We must be able to distinguish between no-tail and the tail at 0:0,
 * therefore increment the cpu number by one.
 */

static inline u32 encode_tail(int cpu, int idx)
{
	u32 tail;

#ifdef CONFIG_DEBUG_SPINLOCK
	BUG_ON(idx > 3);
#endif
	tail  = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
	tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */

	return tail;
}

static inline struct mcs_spinlock *decode_tail(u32 tail)
{
	int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
	int idx = (tail &  _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;

	return per_cpu_ptr(&mcs_nodes[idx], cpu);
}

114
115
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)

116
117
118
119
/*
 * By using the whole 2nd least significant byte for the pending bit, we
 * can allow better optimization of the lock acquisition for the pending
 * bit holder.
120
121
122
 *
 * This internal structure is also used by the set_locked function which
 * is not restricted to _Q_PENDING_BITS == 8.
123
124
125
126
127
 */
struct __qspinlock {
	union {
		atomic_t val;
#ifdef __LITTLE_ENDIAN
128
129
130
131
132
		struct {
			u8	locked;
			u8	pending;
		};
		struct {
133
134
			u16	locked_pending;
			u16	tail;
135
		};
136
#else
137
		struct {
138
139
140
			u16	tail;
			u16	locked_pending;
		};
141
142
143
144
145
146
		struct {
			u8	reserved[2];
			u8	pending;
			u8	locked;
		};
#endif
147
148
149
	};
};

150
#if _Q_PENDING_BITS == 8
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
/**
 * clear_pending_set_locked - take ownership and clear the pending bit.
 * @lock: Pointer to queued spinlock structure
 *
 * *,1,0 -> *,0,1
 *
 * Lock stealing is not allowed if this function is used.
 */
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
{
	struct __qspinlock *l = (void *)lock;

	WRITE_ONCE(l->locked_pending, _Q_LOCKED_VAL);
}

/*
 * xchg_tail - Put in the new queue tail code word & retrieve previous one
 * @lock : Pointer to queued spinlock structure
 * @tail : The new queue tail code word
 * Return: The previous queue tail code word
 *
 * xchg(lock, tail)
 *
 * p,*,* -> n,*,* ; prev = xchg(lock, node)
 */
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
{
	struct __qspinlock *l = (void *)lock;

180
181
182
183
184
185
	/*
	 * Use release semantics to make sure that the MCS node is properly
	 * initialized before changing the tail code.
	 */
	return (u32)xchg_release(&l->tail,
				 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
186
187
188
189
}

#else /* _Q_PENDING_BITS == 8 */

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
/**
 * clear_pending_set_locked - take ownership and clear the pending bit.
 * @lock: Pointer to queued spinlock structure
 *
 * *,1,0 -> *,0,1
 */
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
{
	atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
}

/**
 * xchg_tail - Put in the new queue tail code word & retrieve previous one
 * @lock : Pointer to queued spinlock structure
 * @tail : The new queue tail code word
 * Return: The previous queue tail code word
 *
 * xchg(lock, tail)
 *
 * p,*,* -> n,*,* ; prev = xchg(lock, node)
 */
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
{
	u32 old, new, val = atomic_read(&lock->val);

	for (;;) {
		new = (val & _Q_LOCKED_PENDING_MASK) | tail;
217
218
219
220
221
		/*
		 * Use release semantics to make sure that the MCS node is
		 * properly initialized before changing the tail code.
		 */
		old = atomic_cmpxchg_release(&lock->val, val, new);
222
223
224
225
226
227
228
		if (old == val)
			break;

		val = old;
	}
	return old;
}
229
#endif /* _Q_PENDING_BITS == 8 */
230

231
232
233
234
235
236
237
238
239
240
241
242
243
/**
 * set_locked - Set the lock bit and own the lock
 * @lock: Pointer to queued spinlock structure
 *
 * *,*,0 -> *,0,1
 */
static __always_inline void set_locked(struct qspinlock *lock)
{
	struct __qspinlock *l = (void *)lock;

	WRITE_ONCE(l->locked, _Q_LOCKED_VAL);
}

244
245
246
247
248
249
250

/*
 * Generate the native code for queued_spin_unlock_slowpath(); provide NOPs for
 * all the PV callbacks.
 */

static __always_inline void __pv_init_node(struct mcs_spinlock *node) { }
251
252
static __always_inline void __pv_wait_node(struct mcs_spinlock *node,
					   struct mcs_spinlock *prev) { }
253
254
static __always_inline void __pv_kick_node(struct qspinlock *lock,
					   struct mcs_spinlock *node) { }
255
256
257
static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
						   struct mcs_spinlock *node)
						   { return 0; }
258
259
260
261
262
263

#define pv_enabled()		false

#define pv_init_node		__pv_init_node
#define pv_wait_node		__pv_wait_node
#define pv_kick_node		__pv_kick_node
264
#define pv_wait_head_or_lock	__pv_wait_head_or_lock
265
266
267
268
269

#ifdef CONFIG_PARAVIRT_SPINLOCKS
#define queued_spin_lock_slowpath	native_queued_spin_lock_slowpath
#endif

270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
/*
 * queued_spin_lock_slowpath() can (load-)ACQUIRE the lock before
 * issuing an _unordered_ store to set _Q_LOCKED_VAL.
 *
 * This means that the store can be delayed, but no later than the
 * store-release from the unlock. This means that simply observing
 * _Q_LOCKED_VAL is not sufficient to determine if the lock is acquired.
 *
 * There are two paths that can issue the unordered store:
 *
 *  (1) clear_pending_set_locked():	*,1,0 -> *,0,1
 *
 *  (2) set_locked():			t,0,0 -> t,0,1 ; t != 0
 *      atomic_cmpxchg_relaxed():	t,0,0 -> 0,0,1
 *
 * However, in both cases we have other !0 state we've set before to queue
 * ourseves:
 *
 * For (1) we have the atomic_cmpxchg_acquire() that set _Q_PENDING_VAL, our
 * load is constrained by that ACQUIRE to not pass before that, and thus must
 * observe the store.
 *
 * For (2) we have a more intersting scenario. We enqueue ourselves using
 * xchg_tail(), which ends up being a RELEASE. This in itself is not
 * sufficient, however that is followed by an smp_cond_acquire() on the same
 * word, giving a RELEASE->ACQUIRE ordering. This again constrains our load and
 * guarantees we must observe that store.
 *
 * Therefore both cases have other !0 state that is observable before the
 * unordered locked byte store comes through. This means we can use that to
 * wait for the lock store, and then wait for an unlock.
 */
#ifndef queued_spin_unlock_wait
void queued_spin_unlock_wait(struct qspinlock *lock)
{
	u32 val;

	for (;;) {
		val = atomic_read(&lock->val);

		if (!val) /* not locked, we're done */
			goto done;

		if (val & _Q_LOCKED_MASK) /* locked, go wait for unlock */
			break;

		/* not locked, but pending, wait until we observe the lock */
		cpu_relax();
	}

	/* any unlock is good */
	while (atomic_read(&lock->val) & _Q_LOCKED_MASK)
		cpu_relax();

done:
	smp_rmb(); /* CTRL + RMB -> ACQUIRE */
}
EXPORT_SYMBOL(queued_spin_unlock_wait);
#endif

330
331
#endif /* _GEN_PV_LOCK_SLOWPATH */

332
333
334
335
336
/**
 * queued_spin_lock_slowpath - acquire the queued spinlock
 * @lock: Pointer to queued spinlock structure
 * @val: Current value of the queued spinlock 32-bit word
 *
337
 * (queue tail, pending bit, lock value)
338
 *
339
340
341
342
343
344
345
346
347
348
349
350
351
 *              fast     :    slow                                  :    unlock
 *                       :                                          :
 * uncontended  (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
 *                       :       | ^--------.------.             /  :
 *                       :       v           \      \            |  :
 * pending               :    (0,1,1) +--> (0,1,0)   \           |  :
 *                       :       | ^--'              |           |  :
 *                       :       v                   |           |  :
 * uncontended           :    (n,x,y) +--> (n,0,0) --'           |  :
 *   queue               :       | ^--'                          |  :
 *                       :       v                               |  :
 * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :
 *   queue               :         ^--'                             :
352
353
354
355
356
357
358
359
360
 */
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
	struct mcs_spinlock *prev, *next, *node;
	u32 new, old, tail;
	int idx;

	BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));

361
362
363
	if (pv_enabled())
		goto queue;

364
	if (virt_spin_lock(lock))
365
366
		return;

367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
	/*
	 * wait for in-progress pending->locked hand-overs
	 *
	 * 0,1,0 -> 0,0,1
	 */
	if (val == _Q_PENDING_VAL) {
		while ((val = atomic_read(&lock->val)) == _Q_PENDING_VAL)
			cpu_relax();
	}

	/*
	 * trylock || pending
	 *
	 * 0,0,0 -> 0,0,1 ; trylock
	 * 0,0,1 -> 0,1,1 ; pending
	 */
	for (;;) {
		/*
		 * If we observe any contention; queue.
		 */
		if (val & ~_Q_LOCKED_MASK)
			goto queue;

		new = _Q_LOCKED_VAL;
		if (val == new)
			new |= _Q_PENDING_VAL;

394
395
396
397
398
		/*
		 * Acquire semantic is required here as the function may
		 * return immediately if the lock was free.
		 */
		old = atomic_cmpxchg_acquire(&lock->val, val, new);
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
		if (old == val)
			break;

		val = old;
	}

	/*
	 * we won the trylock
	 */
	if (new == _Q_LOCKED_VAL)
		return;

	/*
	 * we're pending, wait for the owner to go away.
	 *
	 * *,1,1 -> *,1,0
415
416
417
418
419
	 *
	 * this wait loop must be a load-acquire such that we match the
	 * store-release that clears the locked bit and create lock
	 * sequentiality; this is because not all clear_pending_set_locked()
	 * implementations imply full barriers.
420
	 */
421
	smp_cond_acquire(!(atomic_read(&lock->val) & _Q_LOCKED_MASK));
422
423
424
425
426
427

	/*
	 * take ownership and clear the pending bit.
	 *
	 * *,1,0 -> *,0,1
	 */
428
	clear_pending_set_locked(lock);
429
430
431
432
433
434
435
	return;

	/*
	 * End of pending bit optimistic spinning and beginning of MCS
	 * queuing.
	 */
queue:
436
437
438
439
440
441
442
	node = this_cpu_ptr(&mcs_nodes[0]);
	idx = node->count++;
	tail = encode_tail(smp_processor_id(), idx);

	node += idx;
	node->locked = 0;
	node->next = NULL;
443
	pv_init_node(node);
444
445

	/*
446
447
448
	 * We touched a (possibly) cold cacheline in the per-cpu queue node;
	 * attempt the trylock once more in the hope someone let go while we
	 * weren't watching.
449
	 */
450
451
	if (queued_spin_trylock(lock))
		goto release;
452
453

	/*
454
455
456
457
	 * We have already touched the queueing cacheline; don't bother with
	 * pending stuff.
	 *
	 * p,*,* -> n,*,*
458
	 */
459
	old = xchg_tail(lock, tail);
460
	next = NULL;
461
462
463
464
465

	/*
	 * if there was a previous node; link it and wait until reaching the
	 * head of the waitqueue.
	 */
466
	if (old & _Q_TAIL_MASK) {
467
468
469
		prev = decode_tail(old);
		WRITE_ONCE(prev->next, node);

470
		pv_wait_node(node, prev);
471
		arch_mcs_spin_lock_contended(&node->locked);
472
473
474
475
476
477
478
479
480
481

		/*
		 * While waiting for the MCS lock, the next pointer may have
		 * been set by another lock waiter. We optimistically load
		 * the next pointer & prefetch the cacheline for writing
		 * to reduce latency in the upcoming MCS unlock operation.
		 */
		next = READ_ONCE(node->next);
		if (next)
			prefetchw(next);
482
483
484
	}

	/*
485
486
	 * we're at the head of the waitqueue, wait for the owner & pending to
	 * go away.
487
	 *
488
	 * *,x,y -> *,0,0
489
490
491
492
493
494
	 *
	 * this wait loop must use a load-acquire such that we match the
	 * store-release that clears the locked bit and create lock
	 * sequentiality; this is because the set_locked() function below
	 * does not imply a full barrier.
	 *
495
496
	 * The PV pv_wait_head_or_lock function, if active, will acquire
	 * the lock and return a non-zero value. So we have to skip the
497
	 * smp_cond_acquire() call. As the next PV queue head hasn't been
498
499
500
501
502
503
	 * designated yet, there is no way for the locked value to become
	 * _Q_SLOW_VAL. So both the set_locked() and the
	 * atomic_cmpxchg_relaxed() calls will be safe.
	 *
	 * If PV isn't active, 0 will be returned instead.
	 *
504
	 */
505
506
507
	if ((val = pv_wait_head_or_lock(lock, node)))
		goto locked;

508
	smp_cond_acquire(!((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK));
509

510
locked:
511
512
513
	/*
	 * claim the lock:
	 *
514
515
	 * n,0,0 -> 0,0,1 : lock, uncontended
	 * *,0,0 -> *,0,1 : lock, contended
516
517
518
519
	 *
	 * If the queue head is the only one in the queue (lock value == tail),
	 * clear the tail code and grab the lock. Otherwise, we only need
	 * to grab the lock.
520
521
	 */
	for (;;) {
522
523
		/* In the PV case we might already have _Q_LOCKED_VAL set */
		if ((val & _Q_TAIL_MASK) != tail) {
524
			set_locked(lock);
525
			break;
526
		}
527
		/*
528
		 * The smp_cond_acquire() call above has provided the necessary
529
530
531
532
		 * acquire semantics required for locking. At most two
		 * iterations of this loop may be ran.
		 */
		old = atomic_cmpxchg_relaxed(&lock->val, val, _Q_LOCKED_VAL);
533
534
		if (old == val)
			goto release;	/* No contention */
535
536
537
538
539

		val = old;
	}

	/*
540
	 * contended path; wait for next if not observed yet, release.
541
	 */
542
543
544
545
	if (!next) {
		while (!(next = READ_ONCE(node->next)))
			cpu_relax();
	}
546

547
	arch_mcs_spin_unlock_contended(&next->locked);
548
	pv_kick_node(lock, next);
549
550
551
552
553
554
555
556

release:
	/*
	 * release the node
	 */
	this_cpu_dec(mcs_nodes[0].count);
}
EXPORT_SYMBOL(queued_spin_lock_slowpath);
557
558
559
560
561
562
563
564
565
566
567
568
569

/*
 * Generate the paravirt code for queued_spin_unlock_slowpath().
 */
#if !defined(_GEN_PV_LOCK_SLOWPATH) && defined(CONFIG_PARAVIRT_SPINLOCKS)
#define _GEN_PV_LOCK_SLOWPATH

#undef  pv_enabled
#define pv_enabled()	true

#undef pv_init_node
#undef pv_wait_node
#undef pv_kick_node
570
#undef pv_wait_head_or_lock
571
572
573
574
575
576
577
578

#undef  queued_spin_lock_slowpath
#define queued_spin_lock_slowpath	__pv_queued_spin_lock_slowpath

#include "qspinlock_paravirt.h"
#include "qspinlock.c"

#endif