qspinlock_paravirt.h 1.85 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2
3
4
#ifndef __ASM_QSPINLOCK_PARAVIRT_H
#define __ASM_QSPINLOCK_PARAVIRT_H

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
 * registers. For i386, however, only 1 32-bit register needs to be saved
 * and restored. So an optimized version of __pv_queued_spin_unlock() is
 * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
 */
#ifdef CONFIG_64BIT

PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
#define __pv_queued_spin_unlock	__pv_queued_spin_unlock
#define PV_UNLOCK		"__raw_callee_save___pv_queued_spin_unlock"
#define PV_UNLOCK_SLOWPATH	"__raw_callee_save___pv_queued_spin_unlock_slowpath"

/*
 * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
 * which combines the registers saving trunk and the body of the following
 * C code:
 *
 * void __pv_queued_spin_unlock(struct qspinlock *lock)
 * {
25
 *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
26
27
28
29
30
31
32
33
34
35
36
37
38
 *
 *	if (likely(lockval == _Q_LOCKED_VAL))
 *		return;
 *	pv_queued_spin_unlock_slowpath(lock, lockval);
 * }
 *
 * For x86-64,
 *   rdi = lock              (first argument)
 *   rsi = lockval           (second argument)
 *   rdx = internal variable (set to 0)
 */
asm    (".pushsection .text;"
	".globl " PV_UNLOCK ";"
39
	".type " PV_UNLOCK ", @function;"
40
41
	".align 4,0x90;"
	PV_UNLOCK ": "
42
	FRAME_BEGIN
43
44
45
46
47
48
49
	"push  %rdx;"
	"mov   $0x1,%eax;"
	"xor   %edx,%edx;"
	"lock cmpxchg %dl,(%rdi);"
	"cmp   $0x1,%al;"
	"jne   .slowpath;"
	"pop   %rdx;"
50
	FRAME_END
51
52
53
54
55
56
57
	"ret;"
	".slowpath: "
	"push   %rsi;"
	"movzbl %al,%esi;"
	"call " PV_UNLOCK_SLOWPATH ";"
	"pop    %rsi;"
	"pop    %rdx;"
58
	FRAME_END
59
60
61
62
63
64
65
	"ret;"
	".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
	".popsection");

#else /* CONFIG_64BIT */

extern void __pv_queued_spin_unlock(struct qspinlock *lock);
66
67
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);

68
#endif /* CONFIG_64BIT */
69
#endif