idle_book3s.S 22.3 KB
Newer Older
1
/*
2
3
 *  This file contains idle entry/exit functions for POWER7,
 *  POWER8 and POWER9 CPUs.
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 */

#include <linux/threads.h>
#include <asm/processor.h>
#include <asm/page.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/ppc-opcode.h>
19
#include <asm/hw_irq.h>
20
#include <asm/kvm_book3s_asm.h>
21
#include <asm/opal.h>
22
#include <asm/cpuidle.h>
23
#include <asm/exception-64s.h>
24
#include <asm/book3s/64/mmu-hash.h>
25
#include <asm/mmu.h>
26
27
28

#undef DEBUG

29
30
31
32
/*
 * Use unused space in the interrupt stack to save and restore
 * registers for winkle support.
 */
33
#define _MMCR0	GPR0
34
#define _SDR1	GPR3
35
#define _PTCR	GPR3
36
37
38
39
40
41
42
43
#define _RPR	GPR4
#define _SPURR	GPR5
#define _PURR	GPR6
#define _TSCR	GPR7
#define _DSCR	GPR8
#define _AMOR	GPR9
#define _WORT	GPR10
#define _WORC	GPR11
44
#define _LPCR	GPR12
45

46
#define PSSCR_EC_ESL_MASK_SHIFTED          (PSSCR_EC | PSSCR_ESL) >> 16
47

48
49
	.text

50
51
52
53
54
55
56
57
58
/*
 * Used by threads before entering deep idle states. Saves SPRs
 * in interrupt stack frame
 */
save_sprs_to_stack:
	/*
	 * Note all register i.e per-core, per-subcore or per-thread is saved
	 * here since any thread in the core might wake up first
	 */
59
60
61
62
63
BEGIN_FTR_SECTION
	/*
	 * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
	 * SDR1 here
	 */
64
65
66
67
	mfspr	r3,SPRN_PTCR
	std	r3,_PTCR(r1)
	mfspr	r3,SPRN_LPCR
	std	r3,_LPCR(r1)
68
FTR_SECTION_ELSE
69
70
	mfspr	r3,SPRN_SDR1
	std	r3,_SDR1(r1)
71
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
	mfspr	r3,SPRN_RPR
	std	r3,_RPR(r1)
	mfspr	r3,SPRN_SPURR
	std	r3,_SPURR(r1)
	mfspr	r3,SPRN_PURR
	std	r3,_PURR(r1)
	mfspr	r3,SPRN_TSCR
	std	r3,_TSCR(r1)
	mfspr	r3,SPRN_DSCR
	std	r3,_DSCR(r1)
	mfspr	r3,SPRN_AMOR
	std	r3,_AMOR(r1)
	mfspr	r3,SPRN_WORT
	std	r3,_WORT(r1)
	mfspr	r3,SPRN_WORC
	std	r3,_WORC(r1)

	blr

91
92
93
94
95
/*
 * Used by threads when the lock bit of core_idle_state is set.
 * Threads will spin in HMT_LOW until the lock bit is cleared.
 * r14 - pointer to core_idle_state
 * r15 - used to load contents of core_idle_state
96
 * r9  - used as a temporary variable
97
98
99
100
101
 */

core_idle_lock_held:
	HMT_LOW
3:	lwz	r15,0(r14)
102
	andis.	r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
103
104
105
	bne	3b
	HMT_MEDIUM
	lwarx	r15,0,r14
106
107
	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
	bne-	core_idle_lock_held
108
109
	blr

110
111
/*
 * Pass requested state in r3:
112
 *	r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
113
 *	   - Requested PSSCR value in POWER9
114
 *
115
 * Address of idle handler to branch to in realmode in r4
116
 */
117
pnv_powersave_common:
118
	/* Use r3 to pass state nap/sleep/winkle */
119
120
121
122
123
124
	/* NAP is a state loss, we create a regs frame on the
	 * stack, fill it up with the state we care about and
	 * stick a pointer to it in PACAR1. We really only
	 * need to save PC, some CR bits and the NV GPRs,
	 * but for now an interrupt frame will do.
	 */
125
126
	mtctr	r4

127
128
129
130
131
132
	mflr	r0
	std	r0,16(r1)
	stdu	r1,-INT_FRAME_SIZE(r1)
	std	r0,_LINK(r1)
	std	r0,_NIP(r1)

133
	/* We haven't lost state ... yet */
134
	li	r0,0
135
	stb	r0,PACA_NAPSTATELOST(r13)
136
137
138
139

	/* Continue saving state */
	SAVE_GPR(2, r1)
	SAVE_NVGPRS(r1)
140
141
	mfcr	r5
	std	r5,_CCR(r1)
142
143
	std	r1,PACAR1(r13)

144
145
146
147
148
149
	/*
	 * Go to real mode to do the nap, as required by the architecture.
	 * Also, we need to be in real mode before setting hwthread_state,
	 * because as soon as we do that, another thread can switch
	 * the MMU context to the guest.
	 */
150
	LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
151
152
	mtmsrd	r7,0
	bctr
153

154
155
	.globl pnv_enter_arch207_idle_mode
pnv_enter_arch207_idle_mode:
156
157
158
159
160
161
162
163
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	/* Tell KVM we're entering idle */
	li	r4,KVM_HWTHREAD_IN_IDLE
	/******************************************************/
	/*  N O T E   W E L L    ! ! !    N O T E   W E L L   */
	/* The following store to HSTATE_HWTHREAD_STATE(r13)  */
	/* MUST occur in real mode, i.e. with the MMU off,    */
	/* and the MMU must stay off until we clear this flag */
164
165
	/* and test HSTATE_HWTHREAD_REQ(r13) in               */
	/* pnv_powersave_wakeup in this file.                 */
166
167
168
169
170
171
172
173
174
175
	/* The reason is that another thread can switch the   */
	/* MMU to a guest context whenever this flag is set   */
	/* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on,    */
	/* that would potentially cause this thread to start  */
	/* executing instructions from guest memory in        */
	/* hypervisor mode, leading to a host crash or data   */
	/* corruption, or worse.                              */
	/******************************************************/
	stb	r4,HSTATE_HWTHREAD_STATE(r13)
#endif
176
	stb	r3,PACA_THREAD_IDLE_STATE(r13)
177
178
	cmpwi	cr3,r3,PNV_THREAD_SLEEP
	bge	cr3,2f
179
	IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
180
	/* No return */
181
182
183
184
2:
	/* Sleep or winkle */
	lbz	r7,PACA_THREAD_MASK(r13)
	ld	r14,PACA_CORE_IDLE_STATE_PTR(r13)
185
186
187
188
	li	r5,0
	beq	cr3,3f
	lis	r5,PNV_CORE_IDLE_WINKLE_COUNT@h
3:
189
190
lwarx_loop1:
	lwarx	r15,0,r14
191

192
193
	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
	bnel-	core_idle_lock_held
194

195
	add	r15,r15,r5			/* Add if winkle */
196
197
	andc	r15,r15,r7			/* Clear thread bit */

198
	andi.	r9,r15,PNV_CORE_IDLE_THREAD_BITS
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

/*
 * If cr0 = 0, then current thread is the last thread of the core entering
 * sleep. Last thread needs to execute the hardware bug workaround code if
 * required by the platform.
 * Make the workaround call unconditionally here. The below branch call is
 * patched out when the idle states are discovered if the platform does not
 * require it.
 */
.global pnv_fastsleep_workaround_at_entry
pnv_fastsleep_workaround_at_entry:
	beq	fastsleep_workaround_at_entry

	stwcx.	r15,0,r14
	bne-	lwarx_loop1
	isync

216
217
common_enter: /* common code for all the threads entering sleep or winkle */
	bgt	cr3,enter_winkle
218
	IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
219
220

fastsleep_workaround_at_entry:
221
	oris	r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
222
223
224
225
226
227
228
	stwcx.	r15,0,r14
	bne-	lwarx_loop1
	isync

	/* Fast sleep workaround */
	li	r3,1
	li	r4,1
229
	bl	opal_config_cpu_idle_state
230

231
232
	/* Unlock */
	xoris	r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
233
	lwsync
234
	stw	r15,0(r14)
235
236
	b	common_enter

237
enter_winkle:
238
239
	bl	save_sprs_to_stack

240
	IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
241

242
/*
243
 * r3 - PSSCR value corresponding to the requested stop state.
244
245
 */
power_enter_stop:
246
247
248
249
250
251
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	/* Tell KVM we're entering idle */
	li	r4,KVM_HWTHREAD_IN_IDLE
	/* DO THIS IN REAL MODE!  See comment above. */
	stb	r4,HSTATE_HWTHREAD_STATE(r13)
#endif
252
253
254
255
256
/*
 * Check if we are executing the lite variant with ESL=EC=0
 */
	andis.   r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
	clrldi   r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
257
	bne	 .Lhandle_esl_ec_set
258
259
	IDLE_STATE_ENTER_SEQ(PPC_STOP)
	li	r3,0  /* Since we didn't lose state, return 0 */
260
261
262
263
264
265
266
267
268
269
270
271
272

	/*
	 * pnv_wakeup_noloss() expects r12 to contain the SRR1 value so
	 * it can determine if the wakeup reason is an HMI in
	 * CHECK_HMI_INTERRUPT.
	 *
	 * However, when we wakeup with ESL=0, SRR1 will not contain the wakeup
	 * reason, so there is no point setting r12 to SRR1.
	 *
	 * Further, we clear r12 here, so that we don't accidentally enter the
	 * HMI in pnv_wakeup_noloss() if the value of r12[42:45] == WAKE_HMI.
	 */
	li	r12, 0
273
	b 	pnv_wakeup_noloss
274
275

.Lhandle_esl_ec_set:
276
277
278
279
280
281
282
283
	/*
	 * POWER9 DD2 can incorrectly set PMAO when waking up after a
	 * state-loss idle. Saving and restoring MMCR0 over idle is a
	 * workaround.
	 */
	mfspr	r4,SPRN_MMCR0
	std	r4,_MMCR0(r1)

284
285
286
/*
 * Check if the requested state is a deep idle state.
 */
287
	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
288
289
	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
	cmpd	r3,r4
290
	bge	.Lhandle_deep_stop
291
	IDLE_STATE_ENTER_SEQ_NORET(PPC_STOP)
292
.Lhandle_deep_stop:
293
294
295
296
297
298
299
300
301
302
/*
 * Entering deep idle state.
 * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
 * stack and enter stop
 */
	lbz     r7,PACA_THREAD_MASK(r13)
	ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)

lwarx_loop_stop:
	lwarx   r15,0,r14
303
304
	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
	bnel-	core_idle_lock_held
305
306
307
308
309
310
311
312
	andc    r15,r15,r7                      /* Clear thread bit */

	stwcx.  r15,0,r14
	bne-    lwarx_loop_stop
	isync

	bl	save_sprs_to_stack

313
	IDLE_STATE_ENTER_SEQ_NORET(PPC_STOP)
314

315
316
317
318
319
/*
 * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
 * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
 */
_GLOBAL(power7_idle_insn)
320
	/* Now check if user or arch enabled NAP mode */
321
	LOAD_REG_ADDR(r4, pnv_enter_arch207_idle_mode)
322
	b	pnv_powersave_common
323

324
325
#define CHECK_HMI_INTERRUPT						\
BEGIN_FTR_SECTION_NESTED(66);						\
326
	rlwinm	r0,r12,45-31,0xf;  /* extract wake reason field (P8) */	\
327
FTR_SECTION_ELSE_NESTED(66);						\
328
	rlwinm	r0,r12,45-31,0xe;  /* P7 wake reason field is 3 bits */	\
329
330
ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
	cmpwi	r0,0xa;			/* Hypervisor maintenance ? */	\
331
	bne+	20f;							\
332
333
334
335
	/* Invoke opal call to handle hmi */				\
	ld	r2,PACATOC(r13);					\
	ld	r1,PACAR1(r13);						\
	std	r3,ORIG_GPR3(r1);	/* Save original r3 */		\
336
337
338
	li	r3,0;			/* NULL argument */		\
	bl	hmi_exception_realmode;					\
	nop;								\
339
340
341
	ld	r3,ORIG_GPR3(r1);	/* Restore original r3 */	\
20:	nop;

342
/*
343
344
 * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
 * r3 contains desired PSSCR register value.
345
346
 */
_GLOBAL(power9_idle_stop)
347
	std	r3, PACA_REQ_PSSCR(r13)
348
	mtspr 	SPRN_PSSCR,r3
349
	LOAD_REG_ADDR(r4,power_enter_stop)
350
351
	b	pnv_powersave_common
	/* No return */
352

353
/*
354
355
356
357
358
359
360
361
362
363
364
 * On waking up from stop 0,1,2 with ESL=1 on POWER9 DD1,
 * HSPRG0 will be set to the HSPRG0 value of one of the
 * threads in this core. Thus the value we have in r13
 * may not be this thread's paca pointer.
 *
 * Fortunately, the TIR remains invariant. Since this thread's
 * paca pointer is recorded in all its sibling's paca, we can
 * correctly recover this thread's paca pointer if we
 * know the index of this thread in the core.
 *
 * This index can be obtained from the TIR.
365
 *
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
 * i.e, thread's position in the core = TIR.
 * If this value is i, then this thread's paca is
 * paca->thread_sibling_pacas[i].
 */
power9_dd1_recover_paca:
	mfspr	r4, SPRN_TIR
	/*
	 * Since each entry in thread_sibling_pacas is 8 bytes
	 * we need to left-shift by 3 bits. Thus r4 = i * 8
	 */
	sldi	r4, r4, 3
	/* Get &paca->thread_sibling_pacas[0] in r5 */
	ld	r5, PACA_SIBLING_PACA_PTRS(r13)
	/* Load paca->thread_sibling_pacas[i] into r13 */
	ldx	r13, r4, r5
	SET_PACA(r13)
	/*
	 * Indicate that we have lost NVGPR state
	 * which needs to be restored from the stack.
	 */
	li	r3, 1
387
	stb	r3,PACA_NAPSTATELOST(r13)
388
389
	blr

390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
/*
 * Called from machine check handler for powersave wakeups.
 * Low level machine check processing has already been done. Now just
 * go through the wake up path to get everything in order.
 *
 * r3 - The original SRR1 value.
 * Original SRR[01] have been clobbered.
 * MSR_RI is clear.
 */
.global pnv_powersave_wakeup_mce
pnv_powersave_wakeup_mce:
	/* Set cr3 for pnv_powersave_wakeup */
	rlwinm	r11,r3,47-31,30,31
	cmpwi	cr3,r11,2

	/*
	 * Now put the original SRR1 with SRR1_WAKEMCE_RESVD as the wake
407
	 * reason into r12, which allows reuse of the system reset wakeup
408
409
	 * code without being mistaken for another type of wakeup.
	 */
410
	oris	r12,r3,SRR1_WAKEMCE_RESVD@h
411
412
413

	b	pnv_powersave_wakeup

414
415
/*
 * Called from reset vector for powersave wakeups.
416
 * cr3 - set to gt if waking up with partial/complete hypervisor state loss
417
 * r12 - SRR1
418
 */
419
420
.global pnv_powersave_wakeup
pnv_powersave_wakeup:
421
422
	ld	r2, PACATOC(r13)

423
BEGIN_FTR_SECTION
424
425
426
BEGIN_FTR_SECTION_NESTED(70)
	bl	power9_dd1_recover_paca
END_FTR_SECTION_NESTED_IFSET(CPU_FTR_POWER9_DD1, 70)
427
428
429
430
	bl	pnv_restore_hyp_resource_arch300
FTR_SECTION_ELSE
	bl	pnv_restore_hyp_resource_arch207
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
431
432
433
434

	li	r0,PNV_THREAD_RUNNING
	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */

435
436
	mr	r3,r12

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	li	r0,KVM_HWTHREAD_IN_KERNEL
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
	/* Order setting hwthread_state vs. testing hwthread_req */
	sync
	lbz	r0,HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r0,0
	beq	1f
	b	kvm_start_guest
1:
#endif

	/* Return SRR1 from power7_nap() */
	blt	cr3,pnv_wakeup_noloss
	b	pnv_wakeup_loss

453
/*
454
455
 * Check whether we have woken up with hypervisor state loss.
 * If yes, restore hypervisor state and return back to link.
456
457
458
 *
 * cr3 - set to gt if waking up with partial/complete hypervisor state loss
 */
459
pnv_restore_hyp_resource_arch300:
460
461
	/*
	 * Workaround for POWER9, if we lost resources, the ERAT
462
463
	 * might have been mixed up and needs flushing. We also need
	 * to reload MMCR0 (see comment above).
464
465
466
	 */
	blt	cr3,1f
	PPC_INVALIDATE_ERAT
467
468
469
	ld	r1,PACAR1(r13)
	ld	r4,_MMCR0(r1)
	mtspr	SPRN_MMCR0,r4
470
1:
471
472
473
474
475
476
477
	/*
	 * POWER ISA 3. Use PSSCR to determine if we
	 * are waking up from deep idle state
	 */
	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)

478
479
480
481
482
483
484
485
486
BEGIN_FTR_SECTION_NESTED(71)
	/*
	 * Assume that we are waking up from the state
	 * same as the Requested Level (RL) in the PSSCR
	 * which are Bits 60-63
	 */
	ld	r5,PACA_REQ_PSSCR(r13)
	rldicl  r5,r5,0,60
FTR_SECTION_ELSE_NESTED(71)
487
	/*
488
489
490
	 * 0-3 bits correspond to Power-Saving Level Status
	 * which indicates the idle state we are waking up from
	 */
491
	mfspr	r5, SPRN_PSSCR
492
	rldicl  r5,r5,4,60
493
ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_POWER9_DD1, 71)
494
	cmpd	cr4,r5,r4
495
	bge	cr4,pnv_wakeup_tb_loss /* returns to caller */
496

497
	blr	/* Waking up without hypervisor state loss. */
498

499
500
/* Same calling convention as arch300 */
pnv_restore_hyp_resource_arch207:
501
502
	/*
	 * POWER ISA 2.07 or less.
503
	 * Check if we slept with sleep or winkle.
504
	 */
505
506
507
	lbz	r4,PACA_THREAD_IDLE_STATE(r13)
	cmpwi	cr2,r4,PNV_THREAD_NAP
	bgt	cr2,pnv_wakeup_tb_loss	/* Either sleep or Winkle */
508
509
510
511
512
513
514
515

	/*
	 * We fall through here if PACA_THREAD_IDLE_STATE shows we are waking
	 * up from nap. At this stage CR3 shouldn't contains 'gt' since that
	 * indicates we are waking with hypervisor state loss from nap.
	 */
	bgt	cr3,.

516
	blr	/* Waking up without hypervisor state loss */
517

518
519
520
521
522
523
524
525
/*
 * Called if waking up from idle state which can cause either partial or
 * complete hyp state loss.
 * In POWER8, called if waking up from fastsleep or winkle
 * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
 *
 * r13 - PACA
 * cr3 - gt if waking up with partial/complete hypervisor state loss
526
527
 *
 * If ISA300:
528
 * cr4 - gt or eq if waking up from complete hypervisor state loss.
529
530
531
 *
 * If ISA207:
 * r4 - PACA_THREAD_IDLE_STATE
532
 */
533
pnv_wakeup_tb_loss:
534
	ld	r1,PACAR1(r13)
535
	/*
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
	 * Before entering any idle state, the NVGPRs are saved in the stack.
	 * If there was a state loss, or PACA_NAPSTATELOST was set, then the
	 * NVGPRs are restored. If we are here, it is likely that state is lost,
	 * but not guaranteed -- neither ISA207 nor ISA300 tests to reach
	 * here are the same as the test to restore NVGPRS:
	 * PACA_THREAD_IDLE_STATE test for ISA207, PSSCR test for ISA300,
	 * and SRR1 test for restoring NVGPRs.
	 *
	 * We are about to clobber NVGPRs now, so set NAPSTATELOST to
	 * guarantee they will always be restored. This might be tightened
	 * with careful reading of specs (particularly for ISA300) but this
	 * is already a slow wakeup path and it's simpler to be safe.
	 */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)

	/*
553
	 *
554
	 * Save SRR1 and LR in NVGPRs as they might be clobbered in
555
	 * opal_call() (called in CHECK_HMI_INTERRUPT). SRR1 is required
556
557
558
	 * to determine the wakeup reason if we branch to kvm_start_guest. LR
	 * is required to return back to reset vector after hypervisor state
	 * restore is complete.
559
	 */
560
	mr	r19,r12
561
	mr	r18,r4
562
	mflr	r17
563
564
565
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
566
567

	ld	r14,PACA_CORE_IDLE_STATE_PTR(r13)
568
569
	lbz	r7,PACA_THREAD_MASK(r13)

570
	/*
571
572
	 * Take the core lock to synchronize against other threads.
	 *
573
574
575
576
577
578
579
	 * Lock bit is set in one of the 2 cases-
	 * a. In the sleep/winkle enter path, the last thread is executing
	 * fastsleep workaround code.
	 * b. In the wake up path, another thread is executing fastsleep
	 * workaround undo code or resyncing timebase or restoring context
	 * In either case loop until the lock bit is cleared.
	 */
580
581
582
1:
	lwarx	r15,0,r14
	andis.	r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
583
	bnel-	core_idle_lock_held
584
585
586
587
	oris	r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
	stwcx.	r15,0,r14
	bne-	1b
	isync
588

589
590
	andi.	r9,r15,PNV_CORE_IDLE_THREAD_BITS
	cmpwi	cr2,r9,0
591
592
593

	/*
	 * At this stage
594
595
	 * cr2 - eq if first thread to wakeup in core
	 * cr3-  gt if waking up with partial/complete hypervisor state loss
596
	 * ISA300:
597
	 * cr4 - gt or eq if waking up from complete hypervisor state loss.
598
599
	 */

600
BEGIN_FTR_SECTION
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
	/*
	 * Were we in winkle?
	 * If yes, check if all threads were in winkle, decrement our
	 * winkle count, set all thread winkle bits if all were in winkle.
	 * Check if our thread has a winkle bit set, and set cr4 accordingly
	 * (to match ISA300, above). Pseudo-code for core idle state
	 * transitions for ISA207 is as follows (everything happens atomically
	 * due to store conditional and/or lock bit):
	 *
	 * nap_idle() { }
	 * nap_wake() { }
	 *
	 * sleep_idle()
	 * {
	 *	core_idle_state &= ~thread_in_core
	 * }
	 *
	 * sleep_wake()
	 * {
	 *     bool first_in_core, first_in_subcore;
	 *
	 *     first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0;
	 *     first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0;
	 *
	 *     core_idle_state |= thread_in_core;
	 * }
	 *
	 * winkle_idle()
	 * {
	 *	core_idle_state &= ~thread_in_core;
	 *	core_idle_state += 1 << WINKLE_COUNT_SHIFT;
	 * }
	 *
	 * winkle_wake()
	 * {
	 *     bool first_in_core, first_in_subcore, winkle_state_lost;
	 *
	 *     first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0;
	 *     first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0;
	 *
	 *     core_idle_state |= thread_in_core;
	 *
	 *     if ((core_idle_state & WINKLE_MASK) == (8 << WINKLE_COUNT_SIHFT))
	 *         core_idle_state |= THREAD_WINKLE_BITS;
	 *     core_idle_state -= 1 << WINKLE_COUNT_SHIFT;
	 *
	 *     winkle_state_lost = core_idle_state &
	 *				(thread_in_core << WINKLE_THREAD_SHIFT);
	 *     core_idle_state &= ~(thread_in_core << WINKLE_THREAD_SHIFT);
	 * }
	 *
	 */
	cmpwi	r18,PNV_THREAD_WINKLE
	bne	2f
	andis.	r9,r15,PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT@h
	subis	r15,r15,PNV_CORE_IDLE_WINKLE_COUNT@h
	beq	2f
	ori	r15,r15,PNV_CORE_IDLE_THREAD_WINKLE_BITS /* all were winkle */
2:
	/* Shift thread bit to winkle mask, then test if this thread is set,
	 * and remove it from the winkle bits */
	slwi	r8,r7,8
	and	r8,r8,r15
	andc	r15,r15,r8
	cmpwi	cr4,r8,1 /* cr4 will be gt if our bit is set, lt if not */

667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
	lbz	r4,PACA_SUBCORE_SIBLING_MASK(r13)
	and	r4,r4,r15
	cmpwi	r4,0	/* Check if first in subcore */

	or	r15,r15,r7		/* Set thread bit */
	beq	first_thread_in_subcore
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)

	or	r15,r15,r7		/* Set thread bit */
	beq	cr2,first_thread_in_core

	/* Not first thread in core or subcore to wake up */
	b	clear_lock

first_thread_in_subcore:
682
683
684
685
	/*
	 * If waking up from sleep, subcore state is not lost. Hence
	 * skip subcore state restore
	 */
686
	blt	cr4,subcore_state_restored
687
688
689
690

	/* Restore per-subcore state */
	ld      r4,_SDR1(r1)
	mtspr   SPRN_SDR1,r4
691

692
693
694
695
696
697
698
699
700
701
702
703
704
705
	ld      r4,_RPR(r1)
	mtspr   SPRN_RPR,r4
	ld	r4,_AMOR(r1)
	mtspr	SPRN_AMOR,r4

subcore_state_restored:
	/*
	 * Check if the thread is also the first thread in the core. If not,
	 * skip to clear_lock.
	 */
	bne	cr2,clear_lock

first_thread_in_core:

706
	/*
707
708
	 * First thread in the core waking up from any state which can cause
	 * partial or complete hypervisor state loss. It needs to
709
710
	 * call the fastsleep workaround code if the platform requires it.
	 * Call it unconditionally here. The below branch instruction will
711
712
713
	 * be patched out if the platform does not have fastsleep or does not
	 * require the workaround. Patching will be performed during the
	 * discovery of idle-states.
714
715
716
717
718
719
	 */
.global pnv_fastsleep_workaround_at_exit
pnv_fastsleep_workaround_at_exit:
	b	fastsleep_workaround_at_exit

timebase_resync:
720
721
722
723
	/*
	 * Use cr3 which indicates that we are waking up with atleast partial
	 * hypervisor state loss to determine if TIMEBASE RESYNC is needed.
	 */
724
	ble	cr3,.Ltb_resynced
725
	/* Time base re-sync */
726
	bl	opal_resync_timebase;
727
	/*
728
729
	 * If waking up from sleep (POWER8), per core state
	 * is not lost, skip to clear_lock.
730
	 */
731
.Ltb_resynced:
732
	blt	cr4,clear_lock
733

734
735
736
737
738
739
740
741
742
743
744
745
	/*
	 * First thread in the core to wake up and its waking up with
	 * complete hypervisor state loss. Restore per core hypervisor
	 * state.
	 */
BEGIN_FTR_SECTION
	ld	r4,_PTCR(r1)
	mtspr	SPRN_PTCR,r4
	ld	r4,_RPR(r1)
	mtspr	SPRN_RPR,r4
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)

746
747
748
749
750
	ld	r4,_TSCR(r1)
	mtspr	SPRN_TSCR,r4
	ld	r4,_WORC(r1)
	mtspr	SPRN_WORC,r4

751
clear_lock:
752
	xoris	r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
753
754
755
756
	lwsync
	stw	r15,0(r14)

common_exit:
757
758
759
760
761
762
	/*
	 * Common to all threads.
	 *
	 * If waking up from sleep, hypervisor state is not lost. Hence
	 * skip hypervisor state restore.
	 */
763
	blt	cr4,hypervisor_state_restored
764
765
766

	/* Waking up from winkle */

767
768
BEGIN_MMU_FTR_SECTION
	b	no_segments
769
END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
770
771
772
773
774
775
776
777
778
779
780
781
782
	/* Restore SLB  from PACA */
	ld	r8,PACA_SLBSHADOWPTR(r13)

	.rept	SLB_NUM_BOLTED
	li	r3, SLBSHADOW_SAVEAREA
	LDX_BE	r5, r8, r3
	addi	r3, r3, 8
	LDX_BE	r6, r8, r3
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r8,r8,16
	.endr
783
784
785
no_segments:

	/* Restore per thread state */
786
787
788
789
790
791
792
793
794
795

	ld	r4,_SPURR(r1)
	mtspr	SPRN_SPURR,r4
	ld	r4,_PURR(r1)
	mtspr	SPRN_PURR,r4
	ld	r4,_DSCR(r1)
	mtspr	SPRN_DSCR,r4
	ld	r4,_WORT(r1)
	mtspr	SPRN_WORT,r4

796
797
798
799
800
801
802
803
804
805
	/* Call cur_cpu_spec->cpu_restore() */
	LOAD_REG_ADDR(r4, cur_cpu_spec)
	ld	r4,0(r4)
	ld	r12,CPU_SPEC_RESTORE(r4)
#ifdef PPC64_ELF_ABI_v1
	ld	r12,0(r12)
#endif
	mtctr	r12
	bctrl

806
807
808
809
BEGIN_FTR_SECTION
	ld	r4,_LPCR(r1)
	mtspr	SPRN_LPCR,r4
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
810
811
hypervisor_state_restored:

812
	mr	r12,r19
813
	mtlr	r17
814
	blr		/* return to pnv_powersave_wakeup */
815

816
817
818
fastsleep_workaround_at_exit:
	li	r3,1
	li	r4,0
819
	bl	opal_config_cpu_idle_state
820
821
	b	timebase_resync

822
823
824
/*
 * R3 here contains the value that will be returned to the caller
 * of power7_nap.
825
 * R12 contains SRR1 for CHECK_HMI_INTERRUPT.
826
 */
827
828
.global pnv_wakeup_loss
pnv_wakeup_loss:
829
	ld	r1,PACAR1(r13)
830
831
832
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
833
834
	REST_NVGPRS(r1)
	REST_GPR(2, r1)
835
836
	ld	r4,PACAKMSR(r13)
	ld	r5,_LINK(r1)
837
	ld	r6,_CCR(r1)
838
	addi	r1,r1,INT_FRAME_SIZE
839
	mtlr	r5
840
	mtcr	r6
841
842
	mtmsrd	r4
	blr
843

844
845
846
/*
 * R3 here contains the value that will be returned to the caller
 * of power7_nap.
847
 * R12 contains SRR1 for CHECK_HMI_INTERRUPT.
848
 */
849
pnv_wakeup_noloss:
850
851
	lbz	r0,PACA_NAPSTATELOST(r13)
	cmpwi	r0,0
852
	bne	pnv_wakeup_loss
853
	ld	r1,PACAR1(r13)
854
855
856
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
857
	ld	r4,PACAKMSR(r13)
858
	ld	r5,_NIP(r1)
859
	ld	r6,_CCR(r1)
860
	addi	r1,r1,INT_FRAME_SIZE
861
	mtlr	r5
862
	mtcr	r6
863
864
	mtmsrd	r4
	blr