qspinlock.c 17.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
 * Queued spinlock
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
 * (C) Copyright 2013-2014 Red Hat, Inc.
 * (C) Copyright 2015 Intel Corp.
17
 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
18
 *
19
 * Authors: Waiman Long <waiman.long@hpe.com>
20
21
 *          Peter Zijlstra <peterz@infradead.org>
 */
22
23
24

#ifndef _GEN_PV_LOCK_SLOWPATH

25
26
27
28
29
30
#include <linux/smp.h>
#include <linux/bug.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
31
#include <asm/byteorder.h>
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#include <asm/qspinlock.h>

/*
 * The basic principle of a queue-based spinlock can best be understood
 * by studying a classic queue-based spinlock implementation called the
 * MCS lock. The paper below provides a good description for this kind
 * of lock.
 *
 * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
 *
 * This queued spinlock implementation is based on the MCS lock, however to make
 * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
 * API, we must modify it somehow.
 *
 * In particular; where the traditional MCS lock consists of a tail pointer
 * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
 * unlock the next pending (next->locked), we compress both these: {tail,
 * next->locked} into a single u32 value.
 *
 * Since a spinlock disables recursion of its own context and there is a limit
 * to the contexts that can nest; namely: task, softirq, hardirq, nmi. As there
 * are at most 4 nesting levels, it can be encoded by a 2-bit number. Now
 * we can encode the tail by combining the 2-bit nesting level with the cpu
 * number. With one byte for the lock value and 3 bytes for the tail, only a
 * 32-bit word is now needed. Even though we only need 1 bit for the lock,
 * we extend it to a full byte to achieve better performance for architectures
 * that support atomic byte write.
 *
 * We also change the first spinner to spin on the lock bit instead of its
 * node; whereby avoiding the need to carry a node from lock to unlock, and
 * preserving existing lock API. This also makes the unlock code simpler and
 * faster.
64
65
66
67
 *
 * N.B. The current implementation only supports architectures that allow
 *      atomic operations on smaller 8-bit and 16-bit data types.
 *
68
69
70
71
 */

#include "mcs_spinlock.h"

72
73
74
75
76
77
#ifdef CONFIG_PARAVIRT_SPINLOCKS
#define MAX_NODES	8
#else
#define MAX_NODES	4
#endif

78
79
80
81
82
/*
 * Per-CPU queue node structures; we can never have more than 4 nested
 * contexts: task, softirq, hardirq, nmi.
 *
 * Exactly fits one 64-byte cacheline on a 64-bit architecture.
83
84
 *
 * PV doubles the storage and uses the second cacheline for PV state.
85
 */
86
static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
87
88
89
90
91
92

/*
 * We must be able to distinguish between no-tail and the tail at 0:0,
 * therefore increment the cpu number by one.
 */

93
static inline __pure u32 encode_tail(int cpu, int idx)
94
95
96
97
98
99
100
101
102
103
104
105
{
	u32 tail;

#ifdef CONFIG_DEBUG_SPINLOCK
	BUG_ON(idx > 3);
#endif
	tail  = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
	tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */

	return tail;
}

106
static inline __pure struct mcs_spinlock *decode_tail(u32 tail)
107
108
109
110
111
112
113
{
	int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
	int idx = (tail &  _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;

	return per_cpu_ptr(&mcs_nodes[idx], cpu);
}

114
115
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)

116
117
118
119
/*
 * By using the whole 2nd least significant byte for the pending bit, we
 * can allow better optimization of the lock acquisition for the pending
 * bit holder.
120
121
122
 *
 * This internal structure is also used by the set_locked function which
 * is not restricted to _Q_PENDING_BITS == 8.
123
124
125
126
127
 */
struct __qspinlock {
	union {
		atomic_t val;
#ifdef __LITTLE_ENDIAN
128
129
130
131
132
		struct {
			u8	locked;
			u8	pending;
		};
		struct {
133
134
			u16	locked_pending;
			u16	tail;
135
		};
136
#else
137
		struct {
138
139
140
			u16	tail;
			u16	locked_pending;
		};
141
142
143
144
145
146
		struct {
			u8	reserved[2];
			u8	pending;
			u8	locked;
		};
#endif
147
148
149
	};
};

150
#if _Q_PENDING_BITS == 8
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
/**
 * clear_pending_set_locked - take ownership and clear the pending bit.
 * @lock: Pointer to queued spinlock structure
 *
 * *,1,0 -> *,0,1
 *
 * Lock stealing is not allowed if this function is used.
 */
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
{
	struct __qspinlock *l = (void *)lock;

	WRITE_ONCE(l->locked_pending, _Q_LOCKED_VAL);
}

/*
 * xchg_tail - Put in the new queue tail code word & retrieve previous one
 * @lock : Pointer to queued spinlock structure
 * @tail : The new queue tail code word
 * Return: The previous queue tail code word
 *
 * xchg(lock, tail)
 *
 * p,*,* -> n,*,* ; prev = xchg(lock, node)
 */
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
{
	struct __qspinlock *l = (void *)lock;

180
181
182
183
184
185
	/*
	 * Use release semantics to make sure that the MCS node is properly
	 * initialized before changing the tail code.
	 */
	return (u32)xchg_release(&l->tail,
				 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
186
187
188
189
}

#else /* _Q_PENDING_BITS == 8 */

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
/**
 * clear_pending_set_locked - take ownership and clear the pending bit.
 * @lock: Pointer to queued spinlock structure
 *
 * *,1,0 -> *,0,1
 */
static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
{
	atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
}

/**
 * xchg_tail - Put in the new queue tail code word & retrieve previous one
 * @lock : Pointer to queued spinlock structure
 * @tail : The new queue tail code word
 * Return: The previous queue tail code word
 *
 * xchg(lock, tail)
 *
 * p,*,* -> n,*,* ; prev = xchg(lock, node)
 */
static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
{
	u32 old, new, val = atomic_read(&lock->val);

	for (;;) {
		new = (val & _Q_LOCKED_PENDING_MASK) | tail;
217
218
219
220
221
		/*
		 * Use release semantics to make sure that the MCS node is
		 * properly initialized before changing the tail code.
		 */
		old = atomic_cmpxchg_release(&lock->val, val, new);
222
223
224
225
226
227
228
		if (old == val)
			break;

		val = old;
	}
	return old;
}
229
#endif /* _Q_PENDING_BITS == 8 */
230

231
232
233
234
235
236
237
238
239
240
241
242
243
/**
 * set_locked - Set the lock bit and own the lock
 * @lock: Pointer to queued spinlock structure
 *
 * *,*,0 -> *,0,1
 */
static __always_inline void set_locked(struct qspinlock *lock)
{
	struct __qspinlock *l = (void *)lock;

	WRITE_ONCE(l->locked, _Q_LOCKED_VAL);
}

244
245
246
247
248
249
250

/*
 * Generate the native code for queued_spin_unlock_slowpath(); provide NOPs for
 * all the PV callbacks.
 */

static __always_inline void __pv_init_node(struct mcs_spinlock *node) { }
251
252
static __always_inline void __pv_wait_node(struct mcs_spinlock *node,
					   struct mcs_spinlock *prev) { }
253
254
static __always_inline void __pv_kick_node(struct qspinlock *lock,
					   struct mcs_spinlock *node) { }
255
256
257
static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
						   struct mcs_spinlock *node)
						   { return 0; }
258
259
260
261
262
263

#define pv_enabled()		false

#define pv_init_node		__pv_init_node
#define pv_wait_node		__pv_wait_node
#define pv_kick_node		__pv_kick_node
264
#define pv_wait_head_or_lock	__pv_wait_head_or_lock
265
266
267
268
269

#ifdef CONFIG_PARAVIRT_SPINLOCKS
#define queued_spin_lock_slowpath	native_queued_spin_lock_slowpath
#endif

270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
/*
 * Various notes on spin_is_locked() and spin_unlock_wait(), which are
 * 'interesting' functions:
 *
 * PROBLEM: some architectures have an interesting issue with atomic ACQUIRE
 * operations in that the ACQUIRE applies to the LOAD _not_ the STORE (ARM64,
 * PPC). Also qspinlock has a similar issue per construction, the setting of
 * the locked byte can be unordered acquiring the lock proper.
 *
 * This gets to be 'interesting' in the following cases, where the /should/s
 * end up false because of this issue.
 *
 *
 * CASE 1:
 *
 * So the spin_is_locked() correctness issue comes from something like:
 *
 *   CPU0				CPU1
 *
 *   global_lock();			local_lock(i)
 *     spin_lock(&G)			  spin_lock(&L[i])
 *     for (i)				  if (!spin_is_locked(&G)) {
 *       spin_unlock_wait(&L[i]);	    smp_acquire__after_ctrl_dep();
 *					    return;
 *					  }
 *					  // deal with fail
 *
 * Where it is important CPU1 sees G locked or CPU0 sees L[i] locked such
 * that there is exclusion between the two critical sections.
 *
 * The load from spin_is_locked(&G) /should/ be constrained by the ACQUIRE from
 * spin_lock(&L[i]), and similarly the load(s) from spin_unlock_wait(&L[i])
 * /should/ be constrained by the ACQUIRE from spin_lock(&G).
 *
 * Similarly, later stuff is constrained by the ACQUIRE from CTRL+RMB.
 *
 *
 * CASE 2:
 *
 * For spin_unlock_wait() there is a second correctness issue, namely:
 *
 *   CPU0				CPU1
 *
 *   flag = set;
 *   smp_mb();				spin_lock(&l)
 *   spin_unlock_wait(&l);		if (!flag)
 *					  // add to lockless list
 *					spin_unlock(&l);
 *   // iterate lockless list
 *
 * Which wants to ensure that CPU1 will stop adding bits to the list and CPU0
 * will observe the last entry on the list (if spin_unlock_wait() had ACQUIRE
 * semantics etc..)
 *
 * Where flag /should/ be ordered against the locked store of l.
 */

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
/*
 * queued_spin_lock_slowpath() can (load-)ACQUIRE the lock before
 * issuing an _unordered_ store to set _Q_LOCKED_VAL.
 *
 * This means that the store can be delayed, but no later than the
 * store-release from the unlock. This means that simply observing
 * _Q_LOCKED_VAL is not sufficient to determine if the lock is acquired.
 *
 * There are two paths that can issue the unordered store:
 *
 *  (1) clear_pending_set_locked():	*,1,0 -> *,0,1
 *
 *  (2) set_locked():			t,0,0 -> t,0,1 ; t != 0
 *      atomic_cmpxchg_relaxed():	t,0,0 -> 0,0,1
 *
 * However, in both cases we have other !0 state we've set before to queue
 * ourseves:
 *
 * For (1) we have the atomic_cmpxchg_acquire() that set _Q_PENDING_VAL, our
 * load is constrained by that ACQUIRE to not pass before that, and thus must
 * observe the store.
 *
 * For (2) we have a more intersting scenario. We enqueue ourselves using
 * xchg_tail(), which ends up being a RELEASE. This in itself is not
 * sufficient, however that is followed by an smp_cond_acquire() on the same
 * word, giving a RELEASE->ACQUIRE ordering. This again constrains our load and
 * guarantees we must observe that store.
 *
 * Therefore both cases have other !0 state that is observable before the
 * unordered locked byte store comes through. This means we can use that to
 * wait for the lock store, and then wait for an unlock.
 */
#ifndef queued_spin_unlock_wait
void queued_spin_unlock_wait(struct qspinlock *lock)
{
	u32 val;

	for (;;) {
		val = atomic_read(&lock->val);

		if (!val) /* not locked, we're done */
			goto done;

		if (val & _Q_LOCKED_MASK) /* locked, go wait for unlock */
			break;

		/* not locked, but pending, wait until we observe the lock */
		cpu_relax();
	}

	/* any unlock is good */
	while (atomic_read(&lock->val) & _Q_LOCKED_MASK)
		cpu_relax();

done:
382
	smp_acquire__after_ctrl_dep();
383
384
385
386
}
EXPORT_SYMBOL(queued_spin_unlock_wait);
#endif

387
388
#endif /* _GEN_PV_LOCK_SLOWPATH */

389
390
391
392
393
/**
 * queued_spin_lock_slowpath - acquire the queued spinlock
 * @lock: Pointer to queued spinlock structure
 * @val: Current value of the queued spinlock 32-bit word
 *
394
 * (queue tail, pending bit, lock value)
395
 *
396
397
398
399
400
401
402
403
404
405
406
407
408
 *              fast     :    slow                                  :    unlock
 *                       :                                          :
 * uncontended  (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
 *                       :       | ^--------.------.             /  :
 *                       :       v           \      \            |  :
 * pending               :    (0,1,1) +--> (0,1,0)   \           |  :
 *                       :       | ^--'              |           |  :
 *                       :       v                   |           |  :
 * uncontended           :    (n,x,y) +--> (n,0,0) --'           |  :
 *   queue               :       | ^--'                          |  :
 *                       :       v                               |  :
 * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :
 *   queue               :         ^--'                             :
409
410
411
412
413
414
415
416
417
 */
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
	struct mcs_spinlock *prev, *next, *node;
	u32 new, old, tail;
	int idx;

	BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));

418
419
420
	if (pv_enabled())
		goto queue;

421
	if (virt_spin_lock(lock))
422
423
		return;

424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
	/*
	 * wait for in-progress pending->locked hand-overs
	 *
	 * 0,1,0 -> 0,0,1
	 */
	if (val == _Q_PENDING_VAL) {
		while ((val = atomic_read(&lock->val)) == _Q_PENDING_VAL)
			cpu_relax();
	}

	/*
	 * trylock || pending
	 *
	 * 0,0,0 -> 0,0,1 ; trylock
	 * 0,0,1 -> 0,1,1 ; pending
	 */
	for (;;) {
		/*
		 * If we observe any contention; queue.
		 */
		if (val & ~_Q_LOCKED_MASK)
			goto queue;

		new = _Q_LOCKED_VAL;
		if (val == new)
			new |= _Q_PENDING_VAL;

451
452
453
454
455
		/*
		 * Acquire semantic is required here as the function may
		 * return immediately if the lock was free.
		 */
		old = atomic_cmpxchg_acquire(&lock->val, val, new);
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
		if (old == val)
			break;

		val = old;
	}

	/*
	 * we won the trylock
	 */
	if (new == _Q_LOCKED_VAL)
		return;

	/*
	 * we're pending, wait for the owner to go away.
	 *
	 * *,1,1 -> *,1,0
472
473
474
475
476
	 *
	 * this wait loop must be a load-acquire such that we match the
	 * store-release that clears the locked bit and create lock
	 * sequentiality; this is because not all clear_pending_set_locked()
	 * implementations imply full barriers.
477
	 */
478
	smp_cond_load_acquire(&lock->val.counter, !(VAL & _Q_LOCKED_MASK));
479
480
481
482
483
484

	/*
	 * take ownership and clear the pending bit.
	 *
	 * *,1,0 -> *,0,1
	 */
485
	clear_pending_set_locked(lock);
486
487
488
489
490
491
492
	return;

	/*
	 * End of pending bit optimistic spinning and beginning of MCS
	 * queuing.
	 */
queue:
493
494
495
496
497
498
499
	node = this_cpu_ptr(&mcs_nodes[0]);
	idx = node->count++;
	tail = encode_tail(smp_processor_id(), idx);

	node += idx;
	node->locked = 0;
	node->next = NULL;
500
	pv_init_node(node);
501
502

	/*
503
504
505
	 * We touched a (possibly) cold cacheline in the per-cpu queue node;
	 * attempt the trylock once more in the hope someone let go while we
	 * weren't watching.
506
	 */
507
508
	if (queued_spin_trylock(lock))
		goto release;
509
510

	/*
511
512
513
514
	 * We have already touched the queueing cacheline; don't bother with
	 * pending stuff.
	 *
	 * p,*,* -> n,*,*
515
516
	 *
	 * RELEASE, such that the stores to @node must be complete.
517
	 */
518
	old = xchg_tail(lock, tail);
519
	next = NULL;
520
521
522
523
524

	/*
	 * if there was a previous node; link it and wait until reaching the
	 * head of the waitqueue.
	 */
525
	if (old & _Q_TAIL_MASK) {
526
		prev = decode_tail(old);
527
528
529
530
531
532
533
534
535
		/*
		 * The above xchg_tail() is also a load of @lock which generates,
		 * through decode_tail(), a pointer.
		 *
		 * The address dependency matches the RELEASE of xchg_tail()
		 * such that the access to @prev must happen after.
		 */
		smp_read_barrier_depends();

536
537
		WRITE_ONCE(prev->next, node);

538
		pv_wait_node(node, prev);
539
		arch_mcs_spin_lock_contended(&node->locked);
540
541
542
543
544
545
546
547
548
549

		/*
		 * While waiting for the MCS lock, the next pointer may have
		 * been set by another lock waiter. We optimistically load
		 * the next pointer & prefetch the cacheline for writing
		 * to reduce latency in the upcoming MCS unlock operation.
		 */
		next = READ_ONCE(node->next);
		if (next)
			prefetchw(next);
550
551
552
	}

	/*
553
554
	 * we're at the head of the waitqueue, wait for the owner & pending to
	 * go away.
555
	 *
556
	 * *,x,y -> *,0,0
557
558
559
560
561
562
	 *
	 * this wait loop must use a load-acquire such that we match the
	 * store-release that clears the locked bit and create lock
	 * sequentiality; this is because the set_locked() function below
	 * does not imply a full barrier.
	 *
563
564
	 * The PV pv_wait_head_or_lock function, if active, will acquire
	 * the lock and return a non-zero value. So we have to skip the
565
	 * smp_cond_load_acquire() call. As the next PV queue head hasn't been
566
567
568
569
570
571
	 * designated yet, there is no way for the locked value to become
	 * _Q_SLOW_VAL. So both the set_locked() and the
	 * atomic_cmpxchg_relaxed() calls will be safe.
	 *
	 * If PV isn't active, 0 will be returned instead.
	 *
572
	 */
573
574
575
	if ((val = pv_wait_head_or_lock(lock, node)))
		goto locked;

576
	val = smp_cond_load_acquire(&lock->val.counter, !(VAL & _Q_LOCKED_PENDING_MASK));
577

578
locked:
579
580
581
	/*
	 * claim the lock:
	 *
582
583
	 * n,0,0 -> 0,0,1 : lock, uncontended
	 * *,0,0 -> *,0,1 : lock, contended
584
585
586
587
	 *
	 * If the queue head is the only one in the queue (lock value == tail),
	 * clear the tail code and grab the lock. Otherwise, we only need
	 * to grab the lock.
588
589
	 */
	for (;;) {
590
591
		/* In the PV case we might already have _Q_LOCKED_VAL set */
		if ((val & _Q_TAIL_MASK) != tail) {
592
			set_locked(lock);
593
			break;
594
		}
595
		/*
596
597
598
		 * The smp_cond_load_acquire() call above has provided the
		 * necessary acquire semantics required for locking. At most
		 * two iterations of this loop may be ran.
599
600
		 */
		old = atomic_cmpxchg_relaxed(&lock->val, val, _Q_LOCKED_VAL);
601
602
		if (old == val)
			goto release;	/* No contention */
603
604
605
606
607

		val = old;
	}

	/*
608
	 * contended path; wait for next if not observed yet, release.
609
	 */
610
611
612
613
	if (!next) {
		while (!(next = READ_ONCE(node->next)))
			cpu_relax();
	}
614

615
	arch_mcs_spin_unlock_contended(&next->locked);
616
	pv_kick_node(lock, next);
617
618
619
620
621
622
623
624

release:
	/*
	 * release the node
	 */
	this_cpu_dec(mcs_nodes[0].count);
}
EXPORT_SYMBOL(queued_spin_lock_slowpath);
625
626
627
628
629
630
631
632
633
634
635
636
637

/*
 * Generate the paravirt code for queued_spin_unlock_slowpath().
 */
#if !defined(_GEN_PV_LOCK_SLOWPATH) && defined(CONFIG_PARAVIRT_SPINLOCKS)
#define _GEN_PV_LOCK_SLOWPATH

#undef  pv_enabled
#define pv_enabled()	true

#undef pv_init_node
#undef pv_wait_node
#undef pv_kick_node
638
#undef pv_wait_head_or_lock
639
640
641
642
643
644
645
646

#undef  queued_spin_lock_slowpath
#define queued_spin_lock_slowpath	__pv_queued_spin_lock_slowpath

#include "qspinlock_paravirt.h"
#include "qspinlock.c"

#endif