idle_book3s.S 18.6 KB
Newer Older
1
/*
2
3
 *  This file contains idle entry/exit functions for POWER7,
 *  POWER8 and POWER9 CPUs.
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 */

#include <linux/threads.h>
#include <asm/processor.h>
#include <asm/page.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/ppc-opcode.h>
19
#include <asm/hw_irq.h>
20
#include <asm/kvm_book3s_asm.h>
21
#include <asm/opal.h>
22
#include <asm/cpuidle.h>
23
#include <asm/exception-64s.h>
24
#include <asm/book3s/64/mmu-hash.h>
25
#include <asm/mmu.h>
26
27
28

#undef DEBUG

29
30
31
32
33
34
35
36
37
38
39
40
41
/*
 * Use unused space in the interrupt stack to save and restore
 * registers for winkle support.
 */
#define _SDR1	GPR3
#define _RPR	GPR4
#define _SPURR	GPR5
#define _PURR	GPR6
#define _TSCR	GPR7
#define _DSCR	GPR8
#define _AMOR	GPR9
#define _WORT	GPR10
#define _WORC	GPR11
42
43
#define _PTCR	GPR12

44
#define PSSCR_EC_ESL_MASK_SHIFTED          (PSSCR_EC | PSSCR_ESL) >> 16
45

46
47
	.text

48
49
50
51
52
53
54
55
56
/*
 * Used by threads before entering deep idle states. Saves SPRs
 * in interrupt stack frame
 */
save_sprs_to_stack:
	/*
	 * Note all register i.e per-core, per-subcore or per-thread is saved
	 * here since any thread in the core might wake up first
	 */
57
58
59
60
61
62
63
64
BEGIN_FTR_SECTION
	mfspr	r3,SPRN_PTCR
	std	r3,_PTCR(r1)
	/*
	 * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
	 * SDR1 here
	 */
FTR_SECTION_ELSE
65
66
	mfspr	r3,SPRN_SDR1
	std	r3,_SDR1(r1)
67
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
	mfspr	r3,SPRN_RPR
	std	r3,_RPR(r1)
	mfspr	r3,SPRN_SPURR
	std	r3,_SPURR(r1)
	mfspr	r3,SPRN_PURR
	std	r3,_PURR(r1)
	mfspr	r3,SPRN_TSCR
	std	r3,_TSCR(r1)
	mfspr	r3,SPRN_DSCR
	std	r3,_DSCR(r1)
	mfspr	r3,SPRN_AMOR
	std	r3,_AMOR(r1)
	mfspr	r3,SPRN_WORT
	std	r3,_WORT(r1)
	mfspr	r3,SPRN_WORC
	std	r3,_WORC(r1)

	blr

87
88
89
90
91
/*
 * Used by threads when the lock bit of core_idle_state is set.
 * Threads will spin in HMT_LOW until the lock bit is cleared.
 * r14 - pointer to core_idle_state
 * r15 - used to load contents of core_idle_state
92
 * r9  - used as a temporary variable
93
94
95
96
97
98
99
100
101
 */

core_idle_lock_held:
	HMT_LOW
3:	lwz	r15,0(r14)
	andi.   r15,r15,PNV_CORE_IDLE_LOCK_BIT
	bne	3b
	HMT_MEDIUM
	lwarx	r15,0,r14
102
103
	andi.	r9,r15,PNV_CORE_IDLE_LOCK_BIT
	bne	core_idle_lock_held
104
105
	blr

106
107
/*
 * Pass requested state in r3:
108
109
 *	r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
 *	   - Requested STOP state in POWER9
110
111
112
113
 *
 * To check IRQ_HAPPENED in r4
 * 	0 - don't check
 * 	1 - check
114
115
 *
 * Address to 'rfid' to in r5
116
 */
117
pnv_powersave_common:
118
	/* Use r3 to pass state nap/sleep/winkle */
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
	/* NAP is a state loss, we create a regs frame on the
	 * stack, fill it up with the state we care about and
	 * stick a pointer to it in PACAR1. We really only
	 * need to save PC, some CR bits and the NV GPRs,
	 * but for now an interrupt frame will do.
	 */
	mflr	r0
	std	r0,16(r1)
	stdu	r1,-INT_FRAME_SIZE(r1)
	std	r0,_LINK(r1)
	std	r0,_NIP(r1)

	/* Hard disable interrupts */
	mfmsr	r9
	rldicl	r9,r9,48,1
	rotldi	r9,r9,16
	mtmsrd	r9,1			/* hard-disable interrupts */
136
137
138

	/* Check if something happened while soft-disabled */
	lbz	r0,PACAIRQHAPPENED(r13)
139
	andi.	r0,r0,~PACA_IRQ_HARD_DIS@l
140
	beq	1f
141
142
	cmpwi	cr0,r4,0
	beq	1f
143
144
	addi	r1,r1,INT_FRAME_SIZE
	ld	r0,16(r1)
145
	li	r3,0			/* Return 0 (no nap) */
146
147
148
149
150
151
152
153
154
155
156
	mtlr	r0
	blr

1:	/* We mark irqs hard disabled as this is the state we'll
	 * be in when returning and we need to tell arch_local_irq_restore()
	 * about it
	 */
	li	r0,PACA_IRQ_HARD_DIS
	stb	r0,PACAIRQHAPPENED(r13)

	/* We haven't lost state ... yet */
157
	li	r0,0
158
	stb	r0,PACA_NAPSTATELOST(r13)
159
160
161
162

	/* Continue saving state */
	SAVE_GPR(2, r1)
	SAVE_NVGPRS(r1)
163
164
	mfcr	r4
	std	r4,_CCR(r1)
165
166
167
	std	r9,_MSR(r1)
	std	r1,PACAR1(r13)

168
169
170
171
172
173
	/*
	 * Go to real mode to do the nap, as required by the architecture.
	 * Also, we need to be in real mode before setting hwthread_state,
	 * because as soon as we do that, another thread can switch
	 * the MMU context to the guest.
	 */
174
	LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
175
176
177
	li	r6, MSR_RI
	andc	r6, r9, r6
	mtmsrd	r6, 1		/* clear RI before setting SRR0/1 */
178
179
	mtspr	SPRN_SRR0, r5
	mtspr	SPRN_SRR1, r7
180
181
	rfid

182
183
	.globl pnv_enter_arch207_idle_mode
pnv_enter_arch207_idle_mode:
184
185
186
187
188
189
190
191
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	/* Tell KVM we're entering idle */
	li	r4,KVM_HWTHREAD_IN_IDLE
	/******************************************************/
	/*  N O T E   W E L L    ! ! !    N O T E   W E L L   */
	/* The following store to HSTATE_HWTHREAD_STATE(r13)  */
	/* MUST occur in real mode, i.e. with the MMU off,    */
	/* and the MMU must stay off until we clear this flag */
192
193
	/* and test HSTATE_HWTHREAD_REQ(r13) in               */
	/* pnv_powersave_wakeup in this file.                 */
194
195
196
197
198
199
200
201
202
203
	/* The reason is that another thread can switch the   */
	/* MMU to a guest context whenever this flag is set   */
	/* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on,    */
	/* that would potentially cause this thread to start  */
	/* executing instructions from guest memory in        */
	/* hypervisor mode, leading to a host crash or data   */
	/* corruption, or worse.                              */
	/******************************************************/
	stb	r4,HSTATE_HWTHREAD_STATE(r13)
#endif
204
	stb	r3,PACA_THREAD_IDLE_STATE(r13)
205
206
	cmpwi	cr3,r3,PNV_THREAD_SLEEP
	bge	cr3,2f
207
	IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
208
	/* No return */
209
210
211
212
213
214
2:
	/* Sleep or winkle */
	lbz	r7,PACA_THREAD_MASK(r13)
	ld	r14,PACA_CORE_IDLE_STATE_PTR(r13)
lwarx_loop1:
	lwarx	r15,0,r14
215
216
217
218

	andi.   r9,r15,PNV_CORE_IDLE_LOCK_BIT
	bnel	core_idle_lock_held

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
	andc	r15,r15,r7			/* Clear thread bit */

	andi.	r15,r15,PNV_CORE_IDLE_THREAD_BITS

/*
 * If cr0 = 0, then current thread is the last thread of the core entering
 * sleep. Last thread needs to execute the hardware bug workaround code if
 * required by the platform.
 * Make the workaround call unconditionally here. The below branch call is
 * patched out when the idle states are discovered if the platform does not
 * require it.
 */
.global pnv_fastsleep_workaround_at_entry
pnv_fastsleep_workaround_at_entry:
	beq	fastsleep_workaround_at_entry

	stwcx.	r15,0,r14
	bne-	lwarx_loop1
	isync

239
240
common_enter: /* common code for all the threads entering sleep or winkle */
	bgt	cr3,enter_winkle
241
	IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
242
243
244
245
246
247
248
249
250
251

fastsleep_workaround_at_entry:
	ori	r15,r15,PNV_CORE_IDLE_LOCK_BIT
	stwcx.	r15,0,r14
	bne-	lwarx_loop1
	isync

	/* Fast sleep workaround */
	li	r3,1
	li	r4,1
252
	bl	opal_config_cpu_idle_state
253
254
255
256
257
258
259

	/* Clear Lock bit */
	li	r0,0
	lwsync
	stw	r0,0(r14)
	b	common_enter

260
enter_winkle:
261
262
	bl	save_sprs_to_stack

263
	IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
264

265
/*
266
 * r3 - PSSCR value corresponding to the requested stop state.
267
268
 */
power_enter_stop:
269
270
271
272
273
274
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	/* Tell KVM we're entering idle */
	li	r4,KVM_HWTHREAD_IN_IDLE
	/* DO THIS IN REAL MODE!  See comment above. */
	stb	r4,HSTATE_HWTHREAD_STATE(r13)
#endif
275
276
277
278
279
/*
 * Check if we are executing the lite variant with ESL=EC=0
 */
	andis.   r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
	clrldi   r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
280
	bne	 .Lhandle_esl_ec_set
281
282
283
	IDLE_STATE_ENTER_SEQ(PPC_STOP)
	li	r3,0  /* Since we didn't lose state, return 0 */
	b 	pnv_wakeup_noloss
284
285

.Lhandle_esl_ec_set:
286
287
288
/*
 * Check if the requested state is a deep idle state.
 */
289
	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
290
291
	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)
	cmpd	r3,r4
292
	bge	.Lhandle_deep_stop
293
	IDLE_STATE_ENTER_SEQ_NORET(PPC_STOP)
294
.Lhandle_deep_stop:
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
/*
 * Entering deep idle state.
 * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
 * stack and enter stop
 */
	lbz     r7,PACA_THREAD_MASK(r13)
	ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)

lwarx_loop_stop:
	lwarx   r15,0,r14
	andi.   r9,r15,PNV_CORE_IDLE_LOCK_BIT
	bnel    core_idle_lock_held
	andc    r15,r15,r7                      /* Clear thread bit */

	stwcx.  r15,0,r14
	bne-    lwarx_loop_stop
	isync

	bl	save_sprs_to_stack

315
	IDLE_STATE_ENTER_SEQ_NORET(PPC_STOP)
316

317
318
319
320
321
322
_GLOBAL(power7_idle)
	/* Now check if user or arch enabled NAP mode */
	LOAD_REG_ADDRBASE(r3,powersave_nap)
	lwz	r4,ADDROFF(powersave_nap)(r3)
	cmpwi	0,r4,0
	beqlr
323
	li	r3, 1
324
325
326
	/* fall through */

_GLOBAL(power7_nap)
327
	mr	r4,r3
328
	li	r3,PNV_THREAD_NAP
329
	LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
330
	b	pnv_powersave_common
331
332
333
	/* No return */

_GLOBAL(power7_sleep)
334
	li	r3,PNV_THREAD_SLEEP
335
	li	r4,1
336
	LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
337
	b	pnv_powersave_common
338
	/* No return */
339

340
_GLOBAL(power7_winkle)
341
	li	r3,PNV_THREAD_WINKLE
342
	li	r4,1
343
	LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
344
	b	pnv_powersave_common
345
346
	/* No return */

347
348
349
350
351
352
353
354
355
356
357
358
359
#define CHECK_HMI_INTERRUPT						\
	mfspr	r0,SPRN_SRR1;						\
BEGIN_FTR_SECTION_NESTED(66);						\
	rlwinm	r0,r0,45-31,0xf;  /* extract wake reason field (P8) */	\
FTR_SECTION_ELSE_NESTED(66);						\
	rlwinm	r0,r0,45-31,0xe;  /* P7 wake reason field is 3 bits */	\
ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
	cmpwi	r0,0xa;			/* Hypervisor maintenance ? */	\
	bne	20f;							\
	/* Invoke opal call to handle hmi */				\
	ld	r2,PACATOC(r13);					\
	ld	r1,PACAR1(r13);						\
	std	r3,ORIG_GPR3(r1);	/* Save original r3 */		\
360
361
362
	li	r3,0;			/* NULL argument */		\
	bl	hmi_exception_realmode;					\
	nop;								\
363
364
365
	ld	r3,ORIG_GPR3(r1);	/* Restore original r3 */	\
20:	nop;

366
/*
367
368
 * r3 - The PSSCR value corresponding to the stop state.
 * r4 - The PSSCR mask corrresonding to the stop state.
369
370
 */
_GLOBAL(power9_idle_stop)
371
372
373
374
	mfspr   r5,SPRN_PSSCR
	andc    r5,r5,r4
	or      r3,r3,r5
	mtspr 	SPRN_PSSCR,r3
375
	LOAD_REG_ADDR(r5,power_enter_stop)
376
	li	r4,1
377
378
	b	pnv_powersave_common
	/* No return */
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417

/*
 * On waking up from stop 0,1,2 with ESL=1 on POWER9 DD1,
 * HSPRG0 will be set to the HSPRG0 value of one of the
 * threads in this core. Thus the value we have in r13
 * may not be this thread's paca pointer.
 *
 * Fortunately, the TIR remains invariant. Since this thread's
 * paca pointer is recorded in all its sibling's paca, we can
 * correctly recover this thread's paca pointer if we
 * know the index of this thread in the core.
 *
 * This index can be obtained from the TIR.
 *
 * i.e, thread's position in the core = TIR.
 * If this value is i, then this thread's paca is
 * paca->thread_sibling_pacas[i].
 */
power9_dd1_recover_paca:
	mfspr	r4, SPRN_TIR
	/*
	 * Since each entry in thread_sibling_pacas is 8 bytes
	 * we need to left-shift by 3 bits. Thus r4 = i * 8
	 */
	sldi	r4, r4, 3
	/* Get &paca->thread_sibling_pacas[0] in r5 */
	ld	r5, PACA_SIBLING_PACA_PTRS(r13)
	/* Load paca->thread_sibling_pacas[i] into r13 */
	ldx	r13, r4, r5
	SET_PACA(r13)
	ld	r2, PACATOC(r13)
	/*
	 * Indicate that we have lost NVGPR state
	 * which needs to be restored from the stack.
	 */
	li	r3, 1
	stb	r0,PACA_NAPSTATELOST(r13)
	blr

418
419
420
421
/*
 * Called from reset vector for powersave wakeups.
 * cr3 - set to gt if waking up with partial/complete hypervisor state loss
 */
422
423
.global pnv_powersave_wakeup
pnv_powersave_wakeup:
424
425
426
427
428
BEGIN_FTR_SECTION
	bl	pnv_restore_hyp_resource_arch300
FTR_SECTION_ELSE
	bl	pnv_restore_hyp_resource_arch207
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449

	li	r0,PNV_THREAD_RUNNING
	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */

#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	li	r0,KVM_HWTHREAD_IN_KERNEL
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
	/* Order setting hwthread_state vs. testing hwthread_req */
	sync
	lbz	r0,HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r0,0
	beq	1f
	b	kvm_start_guest
1:
#endif

	/* Return SRR1 from power7_nap() */
	mfspr	r3,SPRN_SRR1
	blt	cr3,pnv_wakeup_noloss
	b	pnv_wakeup_loss

450
/*
451
452
 * Check whether we have woken up with hypervisor state loss.
 * If yes, restore hypervisor state and return back to link.
453
454
455
 *
 * cr3 - set to gt if waking up with partial/complete hypervisor state loss
 */
456
457
458
459
460
pnv_restore_hyp_resource_arch300:
	/*
	 * POWER ISA 3. Use PSSCR to determine if we
	 * are waking up from deep idle state
	 */
461
BEGIN_FTR_SECTION
462
463
464
	mflr 	r6
	bl	power9_dd1_recover_paca
	mtlr	r6
465
FTR_SECTION_ELSE
466
	ld	r2, PACATOC(r13)
467
468
ALT_FTR_SECTION_END_IFSET(CPU_FTR_POWER9_DD1)

469
470
471
472
	LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
	ld	r4,ADDROFF(pnv_first_deep_stop_state)(r5)

	mfspr	r5,SPRN_PSSCR
473
	/*
474
475
476
477
478
	 * 0-3 bits correspond to Power-Saving Level Status
	 * which indicates the idle state we are waking up from
	 */
	rldicl  r5,r5,4,60
	cmpd	cr4,r5,r4
479
480
481
	bge	cr4,pnv_wakeup_tb_loss /* returns to caller */

	blr	/* Waking up without hypervisor state loss. */
482

483
484
/* Same calling convention as arch300 */
pnv_restore_hyp_resource_arch207:
485
486
	/*
	 * POWER ISA 2.07 or less.
487
	 * Check if we slept with winkle.
488
	 */
489
490
	ld	r2,PACATOC(r13);

491
492
	lbz	r0,PACA_THREAD_IDLE_STATE(r13)
	cmpwi   cr2,r0,PNV_THREAD_NAP
493
	cmpwi   cr4,r0,PNV_THREAD_WINKLE
494
	bgt     cr2,pnv_wakeup_tb_loss	/* Either sleep or Winkle */
495
496
497
498
499
500
501
502

	/*
	 * We fall through here if PACA_THREAD_IDLE_STATE shows we are waking
	 * up from nap. At this stage CR3 shouldn't contains 'gt' since that
	 * indicates we are waking with hypervisor state loss from nap.
	 */
	bgt	cr3,.

503
	blr	/* Waking up without hypervisor state loss */
504

505
506
507
508
509
510
511
512
/*
 * Called if waking up from idle state which can cause either partial or
 * complete hyp state loss.
 * In POWER8, called if waking up from fastsleep or winkle
 * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
 *
 * r13 - PACA
 * cr3 - gt if waking up with partial/complete hypervisor state loss
513
 * cr4 - gt or eq if waking up from complete hypervisor state loss.
514
 */
515
pnv_wakeup_tb_loss:
516
	ld	r1,PACAR1(r13)
517
518
519
520
521
	/*
	 * Before entering any idle state, the NVGPRs are saved in the stack
	 * and they are restored before switching to the process context. Hence
	 * until they are restored, they are free to be used.
	 *
522
	 * Save SRR1 and LR in NVGPRs as they might be clobbered in
523
	 * opal_call() (called in CHECK_HMI_INTERRUPT). SRR1 is required
524
525
526
	 * to determine the wakeup reason if we branch to kvm_start_guest. LR
	 * is required to return back to reset vector after hypervisor state
	 * restore is complete.
527
	 */
528
	mflr	r17
529
	mfspr	r16,SPRN_SRR1
530
531
532
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
533
534
535
536
537
538
539
540
541
542
543
544
545
546

	lbz	r7,PACA_THREAD_MASK(r13)
	ld	r14,PACA_CORE_IDLE_STATE_PTR(r13)
lwarx_loop2:
	lwarx	r15,0,r14
	andi.	r9,r15,PNV_CORE_IDLE_LOCK_BIT
	/*
	 * Lock bit is set in one of the 2 cases-
	 * a. In the sleep/winkle enter path, the last thread is executing
	 * fastsleep workaround code.
	 * b. In the wake up path, another thread is executing fastsleep
	 * workaround undo code or resyncing timebase or restoring context
	 * In either case loop until the lock bit is cleared.
	 */
547
	bnel	core_idle_lock_held
548
549

	cmpwi	cr2,r15,0
550
551
552

	/*
	 * At this stage
553
554
	 * cr2 - eq if first thread to wakeup in core
	 * cr3-  gt if waking up with partial/complete hypervisor state loss
555
	 * cr4 - gt or eq if waking up from complete hypervisor state loss.
556
557
	 */

558
559
560
561
562
	ori	r15,r15,PNV_CORE_IDLE_LOCK_BIT
	stwcx.	r15,0,r14
	bne-	lwarx_loop2
	isync

563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
BEGIN_FTR_SECTION
	lbz	r4,PACA_SUBCORE_SIBLING_MASK(r13)
	and	r4,r4,r15
	cmpwi	r4,0	/* Check if first in subcore */

	or	r15,r15,r7		/* Set thread bit */
	beq	first_thread_in_subcore
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)

	or	r15,r15,r7		/* Set thread bit */
	beq	cr2,first_thread_in_core

	/* Not first thread in core or subcore to wake up */
	b	clear_lock

first_thread_in_subcore:
579
580
581
582
	/*
	 * If waking up from sleep, subcore state is not lost. Hence
	 * skip subcore state restore
	 */
583
	blt	cr4,subcore_state_restored
584
585
586
587

	/* Restore per-subcore state */
	ld      r4,_SDR1(r1)
	mtspr   SPRN_SDR1,r4
588

589
590
591
592
593
594
595
596
597
598
599
600
601
602
	ld      r4,_RPR(r1)
	mtspr   SPRN_RPR,r4
	ld	r4,_AMOR(r1)
	mtspr	SPRN_AMOR,r4

subcore_state_restored:
	/*
	 * Check if the thread is also the first thread in the core. If not,
	 * skip to clear_lock.
	 */
	bne	cr2,clear_lock

first_thread_in_core:

603
	/*
604
605
	 * First thread in the core waking up from any state which can cause
	 * partial or complete hypervisor state loss. It needs to
606
607
	 * call the fastsleep workaround code if the platform requires it.
	 * Call it unconditionally here. The below branch instruction will
608
609
610
	 * be patched out if the platform does not have fastsleep or does not
	 * require the workaround. Patching will be performed during the
	 * discovery of idle-states.
611
612
613
614
615
616
	 */
.global pnv_fastsleep_workaround_at_exit
pnv_fastsleep_workaround_at_exit:
	b	fastsleep_workaround_at_exit

timebase_resync:
617
618
619
620
	/*
	 * Use cr3 which indicates that we are waking up with atleast partial
	 * hypervisor state loss to determine if TIMEBASE RESYNC is needed.
	 */
621
	ble	cr3,clear_lock
622
	/* Time base re-sync */
623
	bl	opal_resync_timebase;
624
625
626
627
	/*
	 * If waking up from sleep, per core state is not lost, skip to
	 * clear_lock.
	 */
628
	blt	cr4,clear_lock
629

630
631
632
633
634
635
636
637
638
639
640
641
	/*
	 * First thread in the core to wake up and its waking up with
	 * complete hypervisor state loss. Restore per core hypervisor
	 * state.
	 */
BEGIN_FTR_SECTION
	ld	r4,_PTCR(r1)
	mtspr	SPRN_PTCR,r4
	ld	r4,_RPR(r1)
	mtspr	SPRN_RPR,r4
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)

642
643
644
645
646
	ld	r4,_TSCR(r1)
	mtspr	SPRN_TSCR,r4
	ld	r4,_WORC(r1)
	mtspr	SPRN_WORC,r4

647
648
649
650
651
652
clear_lock:
	andi.	r15,r15,PNV_CORE_IDLE_THREAD_BITS
	lwsync
	stw	r15,0(r14)

common_exit:
653
654
655
656
657
658
	/*
	 * Common to all threads.
	 *
	 * If waking up from sleep, hypervisor state is not lost. Hence
	 * skip hypervisor state restore.
	 */
659
	blt	cr4,hypervisor_state_restored
660
661
662

	/* Waking up from winkle */

663
664
BEGIN_MMU_FTR_SECTION
	b	no_segments
665
END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
666
667
668
669
670
671
672
673
674
675
676
677
678
	/* Restore SLB  from PACA */
	ld	r8,PACA_SLBSHADOWPTR(r13)

	.rept	SLB_NUM_BOLTED
	li	r3, SLBSHADOW_SAVEAREA
	LDX_BE	r5, r8, r3
	addi	r3, r3, 8
	LDX_BE	r6, r8, r3
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r8,r8,16
	.endr
679
680
681
no_segments:

	/* Restore per thread state */
682
683
684
685
686
687
688
689
690
691

	ld	r4,_SPURR(r1)
	mtspr	SPRN_SPURR,r4
	ld	r4,_PURR(r1)
	mtspr	SPRN_PURR,r4
	ld	r4,_DSCR(r1)
	mtspr	SPRN_DSCR,r4
	ld	r4,_WORT(r1)
	mtspr	SPRN_WORT,r4

692
693
694
695
696
697
698
699
700
701
	/* Call cur_cpu_spec->cpu_restore() */
	LOAD_REG_ADDR(r4, cur_cpu_spec)
	ld	r4,0(r4)
	ld	r12,CPU_SPEC_RESTORE(r4)
#ifdef PPC64_ELF_ABI_v1
	ld	r12,0(r12)
#endif
	mtctr	r12
	bctrl

702
703
hypervisor_state_restored:

704
	mtspr	SPRN_SRR1,r16
705
	mtlr	r17
706
	blr		/* return to pnv_powersave_wakeup */
707

708
709
710
fastsleep_workaround_at_exit:
	li	r3,1
	li	r4,0
711
	bl	opal_config_cpu_idle_state
712
713
	b	timebase_resync

714
715
716
717
/*
 * R3 here contains the value that will be returned to the caller
 * of power7_nap.
 */
718
719
.global pnv_wakeup_loss
pnv_wakeup_loss:
720
	ld	r1,PACAR1(r13)
721
722
723
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
724
725
	REST_NVGPRS(r1)
	REST_GPR(2, r1)
726
	ld	r6,_CCR(r1)
727
728
729
	ld	r4,_MSR(r1)
	ld	r5,_NIP(r1)
	addi	r1,r1,INT_FRAME_SIZE
730
	mtcr	r6
731
732
733
734
	mtspr	SPRN_SRR1,r4
	mtspr	SPRN_SRR0,r5
	rfid

735
736
737
738
/*
 * R3 here contains the value that will be returned to the caller
 * of power7_nap.
 */
739
pnv_wakeup_noloss:
740
741
	lbz	r0,PACA_NAPSTATELOST(r13)
	cmpwi	r0,0
742
	bne	pnv_wakeup_loss
743
744
745
BEGIN_FTR_SECTION
	CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
746
	ld	r1,PACAR1(r13)
747
	ld	r6,_CCR(r1)
748
749
750
	ld	r4,_MSR(r1)
	ld	r5,_NIP(r1)
	addi	r1,r1,INT_FRAME_SIZE
751
	mtcr	r6
752
753
754
	mtspr	SPRN_SRR1,r4
	mtspr	SPRN_SRR0,r5
	rfid