exceptions-64s.S 44.2 KB
Newer Older
1
2
3
4
5
6
7
/*
 * This file contains the 64-bit "server" PowerPC variant
 * of the low level exception handling including exception
 * vectors, exception return, part of the slb and stab
 * handling and other fixed offset specific things.
 *
 * This file is meant to be #included from head_64.S due to
Lucas De Marchi's avatar
Lucas De Marchi committed
8
 * position dependent assembly.
9
10
11
12
13
14
 *
 * Most of this originates from head_64.S and thus has the same
 * copyright history.
 *
 */

15
#include <asm/hw_irq.h>
16
#include <asm/exception-64s.h>
17
#include <asm/ptrace.h>
18
#include <asm/cpuidle.h>
19
#include <asm/head-64.h>
20

21
/*
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 * There are a few constraints to be concerned with.
 * - Real mode exceptions code/data must be located at their physical location.
 * - Virtual mode exceptions must be mapped at their 0xc000... location.
 * - Fixed location code must not call directly beyond the __end_interrupts
 *   area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
 *   must be used.
 * - LOAD_HANDLER targets must be within first 64K of physical 0 /
 *   virtual 0xc00...
 * - Conditional branch targets must be within +/-32K of caller.
 *
 * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
 * therefore don't have to run in physically located code or rfid to
 * virtual mode kernel code. However on relocatable kernels they do have
 * to branch to KERNELBASE offset because the rest of the kernel (outside
 * the exception vectors) may be located elsewhere.
 *
 * Virtual exceptions correspond with physical, except their entry points
 * are offset by 0xc000000000000000 and also tend to get an added 0x4000
 * offset applied. Virtual exceptions are enabled with the Alternate
 * Interrupt Location (AIL) bit set in the LPCR. However this does not
 * guarantee they will be delivered virtually. Some conditions (see the ISA)
 * cause exceptions to be delivered in real mode.
 *
 * It's impossible to receive interrupts below 0x300 via AIL.
 *
 * KVM: None of the virtual exceptions are from the guest. Anything that
 * escalated to HV=1 from HV=0 is delivered via real mode handlers.
 *
 *
51
52
 * We layout physical memory as follows:
 * 0x0000 - 0x00ff : Secondary processor spin code
53
54
55
56
 * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
 * 0x1900 - 0x3fff : Real mode trampolines
 * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
 * 0x5900 - 0x6fff : Relon mode trampolines
57
 * 0x7000 - 0x7fff : FWNMI data area
58
59
 * 0x8000 -   .... : Common interrupt handlers, remaining early
 *                   setup code, rest of kernel.
60
61
62
63
 *
 * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
 * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
 * vectors there.
64
65
66
67
68
69
70
71
72
73
74
 */
OPEN_FIXED_SECTION(real_vectors,        0x0100, 0x1900)
OPEN_FIXED_SECTION(real_trampolines,    0x1900, 0x4000)
OPEN_FIXED_SECTION(virt_vectors,        0x4000, 0x5900)
OPEN_FIXED_SECTION(virt_trampolines,    0x5900, 0x7000)
#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
/*
 * Data area reserved for FWNMI option.
 * This address (0x7000) is fixed by the RPA.
 * pseries and powernv need to keep the whole page from
 * 0x7000 to 0x8000 free for use by the firmware
75
 */
76
77
78
79
80
81
82
83
ZERO_FIXED_SECTION(fwnmi_page,          0x7000, 0x8000)
OPEN_TEXT_SECTION(0x8000)
#else
OPEN_TEXT_SECTION(0x7000)
#endif

USE_FIXED_SECTION(real_vectors)

84
85
86
87
88
89
90
91
92
93
94
/*
 * This is the start of the interrupt handlers for pSeries
 * This code runs with relocation off.
 * Code from here to __end_interrupts gets copied down to real
 * address 0x100 when we are running a relocatable kernel.
 * Therefore any relative branches in this section must only
 * branch to labels in this section.
 */
	.globl __start_interrupts
__start_interrupts:

95
96
97
/* No virt vectors corresponding with 0x0..0x100 */
EXC_VIRT_NONE(0x4000, 0x4100)

98

99
#ifdef CONFIG_PPC_P7_NAP
100
101
102
	/*
	 * If running native on arch 2.06 or later, check if we are waking up
	 * from nap/sleep/winkle, and branch to idle handler.
103
	 */
104
105
106
107
108
109
110
111
112
113
114
115
#define IDLETEST(n)							\
	BEGIN_FTR_SECTION ;						\
	mfspr	r10,SPRN_SRR1 ;						\
	rlwinm.	r10,r10,47-31,30,31 ;					\
	beq-	1f ;							\
	cmpwi	cr3,r10,2 ;						\
	BRANCH_TO_COMMON(r10, system_reset_idle_common) ;		\
1:									\
	END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
#else
#define IDLETEST NOTEST
#endif
116

117
118
EXC_REAL_BEGIN(system_reset, 0x100, 0x200)
	SET_SCRATCH0(r13)
119
120
121
	GET_PACA(r13)
	clrrdi	r13,r13,1 /* Last bit of HSPRG0 is set if waking from winkle */
	EXCEPTION_PROLOG_PSERIES_PACA(PACA_EXGEN, system_reset_common, EXC_STD,
122
123
124
125
126
127
128
				 IDLETEST, 0x100)

EXC_REAL_END(system_reset, 0x100, 0x200)
EXC_VIRT_NONE(0x4100, 0x4200)

#ifdef CONFIG_PPC_P7_NAP
EXC_COMMON_BEGIN(system_reset_idle_common)
129
130
131
BEGIN_FTR_SECTION
	GET_PACA(r13) /* Restore HSPRG0 to get the winkle bit in r13 */
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
132
	bl	pnv_restore_hyp_resource
133

134
135
	li	r0,PNV_THREAD_RUNNING
	stb	r0,PACA_THREAD_IDLE_STATE(r13)	/* Clear thread state */
136

137
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
138
139
140
141
142
143
144
	li	r0,KVM_HWTHREAD_IN_KERNEL
	stb	r0,HSTATE_HWTHREAD_STATE(r13)
	/* Order setting hwthread_state vs. testing hwthread_req */
	sync
	lbz	r0,HSTATE_HWTHREAD_REQ(r13)
	cmpwi	r0,0
	beq	1f
145
146
147
148
	b	kvm_start_guest
1:
#endif

149
150
	/* Return SRR1 from power7_nap() */
	mfspr	r3,SPRN_SRR1
151
	blt	cr3,2f
152
153
	b	pnv_wakeup_loss
2:	b	pnv_wakeup_noloss
154
#endif
155

156
157
158
159
160
161
162
163
164
165
166
167
EXC_COMMON(system_reset_common, 0x100, system_reset_exception)

#ifdef CONFIG_PPC_PSERIES
/*
 * Vectors for the FWNMI option.  Share common code.
 */
TRAMP_REAL_BEGIN(system_reset_fwnmi)
	SET_SCRATCH0(r13)		/* save r13 */
	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
				 NOTEST, 0x100)
#endif /* CONFIG_PPC_PSERIES */

168

169
EXC_REAL_BEGIN(machine_check, 0x200, 0x300)
170
171
172
173
	/* This is moved out of line as it can be patched by FW, but
	 * some code path might still want to branch into the original
	 * vector
	 */
174
	SET_SCRATCH0(r13)		/* save r13 */
175
176
	/*
	 * Running native on arch 2.06 or later, we may wakeup from winkle
177
	 * inside machine check. If yes, then last bit of HSPRG0 would be set
178
	 * to 1. Hence clear it unconditionally.
179
	 */
180
181
182
	GET_PACA(r13)
	clrrdi	r13,r13,1
	SET_PACA(r13)
183
	EXCEPTION_PROLOG_0(PACA_EXMC)
184
BEGIN_FTR_SECTION
185
	b	machine_check_powernv_early
186
FTR_SECTION_ELSE
187
	b	machine_check_pSeries_0
188
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
189
EXC_REAL_END(machine_check, 0x200, 0x300)
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
EXC_VIRT_NONE(0x4200, 0x4300)
TRAMP_REAL_BEGIN(machine_check_powernv_early)
BEGIN_FTR_SECTION
	EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
	/*
	 * Register contents:
	 * R13		= PACA
	 * R9		= CR
	 * Original R9 to R13 is saved on PACA_EXMC
	 *
	 * Switch to mc_emergency stack and handle re-entrancy (we limit
	 * the nested MCE upto level 4 to avoid stack overflow).
	 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
	 *
	 * We use paca->in_mce to check whether this is the first entry or
	 * nested machine check. We increment paca->in_mce to track nested
	 * machine checks.
	 *
	 * If this is the first entry then set stack pointer to
	 * paca->mc_emergency_sp, otherwise r1 is already pointing to
	 * stack frame on mc_emergency stack.
	 *
	 * NOTE: We are here with MSR_ME=0 (off), which means we risk a
	 * checkstop if we get another machine check exception before we do
	 * rfid with MSR_ME=1.
	 */
	mr	r11,r1			/* Save r1 */
	lhz	r10,PACA_IN_MCE(r13)
	cmpwi	r10,0			/* Are we in nested machine check */
	bne	0f			/* Yes, we are. */
	/* First machine check entry */
	ld	r1,PACAMCEMERGSP(r13)	/* Use MC emergency stack */
0:	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame */
	addi	r10,r10,1		/* increment paca->in_mce */
	sth	r10,PACA_IN_MCE(r13)
	/* Limit nested MCE to level 4 to avoid stack overflow */
	cmpwi	r10,4
	bgt	2f			/* Check if we hit limit of 4 */
	std	r11,GPR1(r1)		/* Save r1 on the stack. */
	std	r11,0(r1)		/* make stack chain pointer */
	mfspr	r11,SPRN_SRR0		/* Save SRR0 */
	std	r11,_NIP(r1)
	mfspr	r11,SPRN_SRR1		/* Save SRR1 */
	std	r11,_MSR(r1)
	mfspr	r11,SPRN_DAR		/* Save DAR */
	std	r11,_DAR(r1)
	mfspr	r11,SPRN_DSISR		/* Save DSISR */
	std	r11,_DSISR(r1)
	std	r9,_CCR(r1)		/* Save CR in stackframe */
	/* Save r9 through r13 from EXMC save area to stack frame. */
	EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
	mfmsr	r11			/* get MSR value */
	ori	r11,r11,MSR_ME		/* turn on ME bit */
	ori	r11,r11,MSR_RI		/* turn on RI bit */
	LOAD_HANDLER(r12, machine_check_handle_early)
1:	mtspr	SPRN_SRR0,r12
	mtspr	SPRN_SRR1,r11
	rfid
	b	.	/* prevent speculative execution */
2:
	/* Stack overflow. Stay on emergency stack and panic.
	 * Keep the ME bit off while panic-ing, so that if we hit
	 * another machine check we checkstop.
	 */
	addi	r1,r1,INT_FRAME_SIZE	/* go back to previous stack frame */
	ld	r11,PACAKMSR(r13)
	LOAD_HANDLER(r12, unrecover_mce)
	li	r10,MSR_ME
	andc	r11,r11,r10		/* Turn off MSR_ME */
	b	1b
	b	.	/* prevent speculative execution */
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)

TRAMP_REAL_BEGIN(machine_check_pSeries)
	.globl machine_check_fwnmi
machine_check_fwnmi:
	SET_SCRATCH0(r13)		/* save r13 */
	EXCEPTION_PROLOG_0(PACA_EXMC)
machine_check_pSeries_0:
	EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST_PR, 0x200)
	/*
	 * The following is essentially EXCEPTION_PROLOG_PSERIES_1 with the
	 * difference that MSR_RI is not enabled, because PACA_EXMC is being
	 * used, so nested machine check corrupts it. machine_check_common
	 * enables MSR_RI.
	 */
	ld	r10,PACAKMSR(r13)
	xori	r10,r10,MSR_RI
	mfspr	r11,SPRN_SRR0
	LOAD_HANDLER(r12, machine_check_common)
	mtspr	SPRN_SRR0,r12
	mfspr	r12,SPRN_SRR1
	mtspr	SPRN_SRR1,r10
	rfid
	b	.	/* prevent speculative execution */

TRAMP_KVM_SKIP(PACA_EXMC, 0x200)

EXC_COMMON_BEGIN(machine_check_common)
	/*
	 * Machine check is different because we use a different
	 * save area: PACA_EXMC instead of PACA_EXGEN.
	 */
	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXMC+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXMC+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
	FINISH_NAP
	RECONCILE_IRQ_STATE(r10, r11)
	ld	r3,PACA_EXMC+EX_DAR(r13)
	lwz	r4,PACA_EXMC+EX_DSISR(r13)
	/* Enable MSR_RI when finished with PACA_EXMC */
	li	r10,MSR_RI
	mtmsrd 	r10,1
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
	bl	save_nvgprs
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	machine_check_exception
	b	ret_from_except

#define MACHINE_CHECK_HANDLER_WINDUP			\
	/* Clear MSR_RI before setting SRR0 and SRR1. */\
	li	r0,MSR_RI;				\
	mfmsr	r9;		/* get MSR value */	\
	andc	r9,r9,r0;				\
	mtmsrd	r9,1;		/* Clear MSR_RI */	\
	/* Move original SRR0 and SRR1 into the respective regs */	\
	ld	r9,_MSR(r1);				\
	mtspr	SPRN_SRR1,r9;				\
	ld	r3,_NIP(r1);				\
	mtspr	SPRN_SRR0,r3;				\
	ld	r9,_CTR(r1);				\
	mtctr	r9;					\
	ld	r9,_XER(r1);				\
	mtxer	r9;					\
	ld	r9,_LINK(r1);				\
	mtlr	r9;					\
	REST_GPR(0, r1);				\
	REST_8GPRS(2, r1);				\
	REST_GPR(10, r1);				\
	ld	r11,_CCR(r1);				\
	mtcr	r11;					\
	/* Decrement paca->in_mce. */			\
	lhz	r12,PACA_IN_MCE(r13);			\
	subi	r12,r12,1;				\
	sth	r12,PACA_IN_MCE(r13);			\
	REST_GPR(11, r1);				\
	REST_2GPRS(12, r1);				\
	/* restore original r1. */			\
	ld	r1,GPR1(r1)

	/*
	 * Handle machine check early in real mode. We come here with
	 * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
	 */
EXC_COMMON_BEGIN(machine_check_handle_early)
	std	r0,GPR0(r1)	/* Save r0 */
	EXCEPTION_PROLOG_COMMON_3(0x200)
	bl	save_nvgprs
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	machine_check_early
	std	r3,RESULT(r1)	/* Save result */
	ld	r12,_MSR(r1)
#ifdef	CONFIG_PPC_P7_NAP
	/*
	 * Check if thread was in power saving mode. We come here when any
	 * of the following is true:
	 * a. thread wasn't in power saving mode
	 * b. thread was in power saving mode with no state loss,
	 *    supervisor state loss or hypervisor state loss.
	 *
	 * Go back to nap/sleep/winkle mode again if (b) is true.
	 */
	rlwinm.	r11,r12,47-31,30,31	/* Was it in power saving mode? */
	beq	4f			/* No, it wasn;t */
	/* Thread was in power saving mode. Go back to nap again. */
	cmpwi	r11,2
	blt	3f
	/* Supervisor/Hypervisor state loss */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)
3:	bl	machine_check_queue_event
	MACHINE_CHECK_HANDLER_WINDUP
	GET_PACA(r13)
	ld	r1,PACAR1(r13)
	/*
	 * Check what idle state this CPU was in and go back to same mode
	 * again.
	 */
	lbz	r3,PACA_THREAD_IDLE_STATE(r13)
	cmpwi	r3,PNV_THREAD_NAP
	bgt	10f
384
	IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
385
386
387
388
	/* No return */
10:
	cmpwi	r3,PNV_THREAD_SLEEP
	bgt	2f
389
	IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
390
391
392
393
394
395
	/* No return */

2:
	/*
	 * Go back to winkle. Please note that this thread was woken up in
	 * machine check from winkle and have not restored the per-subcore
396
	 * state. Hence before going back to winkle, set last bit of HSPRG0
397
398
399
400
401
402
	 * to 1. This will make sure that if this thread gets woken up
	 * again at reset vector 0x100 then it will get chance to restore
	 * the subcore state.
	 */
	ori	r13,r13,1
	SET_PACA(r13)
403
	IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
	/* No return */
4:
#endif
	/*
	 * Check if we are coming from hypervisor userspace. If yes then we
	 * continue in host kernel in V mode to deliver the MC event.
	 */
	rldicl.	r11,r12,4,63		/* See if MC hit while in HV mode. */
	beq	5f
	andi.	r11,r12,MSR_PR		/* See if coming from user. */
	bne	9f			/* continue in V mode if we are. */

5:
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
	/*
	 * We are coming from kernel context. Check if we are coming from
	 * guest. if yes, then we can continue. We will fall through
	 * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
	 */
	lbz	r11,HSTATE_IN_GUEST(r13)
	cmpwi	r11,0			/* Check if coming from guest */
	bne	9f			/* continue if we are. */
#endif
	/*
	 * At this point we are not sure about what context we come from.
	 * Queue up the MCE event and return from the interrupt.
	 * But before that, check if this is an un-recoverable exception.
	 * If yes, then stay on emergency stack and panic.
	 */
	andi.	r11,r12,MSR_RI
	bne	2f
1:	mfspr	r11,SPRN_SRR0
	LOAD_HANDLER(r10,unrecover_mce)
	mtspr	SPRN_SRR0,r10
	ld	r10,PACAKMSR(r13)
	/*
	 * We are going down. But there are chances that we might get hit by
	 * another MCE during panic path and we may run into unstable state
	 * with no way out. Hence, turn ME bit off while going down, so that
	 * when another MCE is hit during panic path, system will checkstop
	 * and hypervisor will get restarted cleanly by SP.
	 */
	li	r3,MSR_ME
	andc	r10,r10,r3		/* Turn off MSR_ME */
	mtspr	SPRN_SRR1,r10
	rfid
	b	.
2:
	/*
	 * Check if we have successfully handled/recovered from error, if not
	 * then stay on emergency stack and panic.
	 */
	ld	r3,RESULT(r1)	/* Load result */
	cmpdi	r3,0		/* see if we handled MCE successfully */

	beq	1b		/* if !handled then panic */
	/*
	 * Return from MC interrupt.
	 * Queue up the MCE event so that we can log it later, while
	 * returning from kernel or opal call.
	 */
	bl	machine_check_queue_event
	MACHINE_CHECK_HANDLER_WINDUP
	rfid
9:
	/* Deliver the machine check to host kernel in V mode. */
	MACHINE_CHECK_HANDLER_WINDUP
	b	machine_check_pSeries

EXC_COMMON_BEGIN(unrecover_mce)
	/* Invoke machine_check_exception to print MCE event and panic. */
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	machine_check_exception
	/*
	 * We will not reach here. Even if we did, there is no way out. Call
	 * unrecoverable_exception and die.
	 */
1:	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	unrecoverable_exception
	b	1b

485

486
EXC_REAL(data_access, 0x300, 0x380)
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
EXC_VIRT(data_access, 0x4300, 0x4380, 0x300)
TRAMP_KVM_SKIP(PACA_EXGEN, 0x300)

EXC_COMMON_BEGIN(data_access_common)
	/*
	 * Here r13 points to the paca, r9 contains the saved CR,
	 * SRR0 and SRR1 are saved in r11 and r12,
	 * r9 - r13 are saved in paca->exgen.
	 */
	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXGEN+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
	RECONCILE_IRQ_STATE(r10, r11)
	ld	r12,_MSR(r1)
	ld	r3,PACA_EXGEN+EX_DAR(r13)
	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
	li	r5,0x300
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
BEGIN_MMU_FTR_SECTION
	b	do_hash_page		/* Try to handle as hpte fault */
MMU_FTR_SECTION_ELSE
	b	handle_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)

514

515
EXC_REAL_BEGIN(data_access_slb, 0x380, 0x400)
516
	SET_SCRATCH0(r13)
517
	EXCEPTION_PROLOG_0(PACA_EXSLB)
518
	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380)
519
520
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_DAR
521
	mfspr	r12,SPRN_SRR1
522
	crset	4*cr6+eq
523
#ifndef CONFIG_RELOCATABLE
524
	b	slb_miss_realmode
525
526
#else
	/*
527
	 * We can't just use a direct branch to slb_miss_realmode
528
529
530
531
	 * because the distance from here to there depends on where
	 * the kernel ends up being put.
	 */
	mfctr	r11
532
	LOAD_HANDLER(r10, slb_miss_realmode)
533
534
535
	mtctr	r10
	bctr
#endif
536
EXC_REAL_END(data_access_slb, 0x380, 0x400)
537

538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x4400)
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXSLB)
	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_DAR
	mfspr	r12,SPRN_SRR1
	crset	4*cr6+eq
#ifndef CONFIG_RELOCATABLE
	b	slb_miss_realmode
#else
	/*
	 * We can't just use a direct branch to slb_miss_realmode
	 * because the distance from here to there depends on where
	 * the kernel ends up being put.
	 */
	mfctr	r11
	LOAD_HANDLER(r10, slb_miss_realmode)
	mtctr	r10
	bctr
#endif
EXC_VIRT_END(data_access_slb, 0x4380, 0x4400)
TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)


563
EXC_REAL(instruction_access, 0x400, 0x480)
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
EXC_VIRT(instruction_access, 0x4400, 0x4480, 0x400)
TRAMP_KVM(PACA_EXGEN, 0x400)

EXC_COMMON_BEGIN(instruction_access_common)
	EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
	RECONCILE_IRQ_STATE(r10, r11)
	ld	r12,_MSR(r1)
	ld	r3,_NIP(r1)
	andis.	r4,r12,0x5820
	li	r5,0x400
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
BEGIN_MMU_FTR_SECTION
	b	do_hash_page		/* Try to handle as hpte fault */
MMU_FTR_SECTION_ELSE
	b	handle_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)

582

583
EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x500)
584
	SET_SCRATCH0(r13)
585
	EXCEPTION_PROLOG_0(PACA_EXSLB)
586
	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
587
588
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
589
	mfspr	r12,SPRN_SRR1
590
	crclr	4*cr6+eq
591
#ifndef CONFIG_RELOCATABLE
592
	b	slb_miss_realmode
593
594
#else
	mfctr	r11
595
	LOAD_HANDLER(r10, slb_miss_realmode)
596
597
598
	mtctr	r10
	bctr
#endif
599
EXC_REAL_END(instruction_access_slb, 0x480, 0x500)
600

601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x4500)
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXSLB)
	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
	std	r3,PACA_EXSLB+EX_R3(r13)
	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
	mfspr	r12,SPRN_SRR1
	crclr	4*cr6+eq
#ifndef CONFIG_RELOCATABLE
	b	slb_miss_realmode
#else
	mfctr	r11
	LOAD_HANDLER(r10, slb_miss_realmode)
	mtctr	r10
	bctr
#endif
EXC_VIRT_END(instruction_access_slb, 0x4480, 0x4500)
TRAMP_KVM(PACA_EXSLB, 0x480)


/* This handler is used by both 0x380 and 0x480 slb miss interrupts */
EXC_COMMON_BEGIN(slb_miss_realmode)
	/*
	 * r13 points to the PACA, r9 contains the saved CR,
	 * r12 contain the saved SRR1, SRR0 is still ready for return
	 * r3 has the faulting address
	 * r9 - r13 are saved in paca->exslb.
	 * r3 is saved in paca->slb_r3
 	 * cr6.eq is set for a D-SLB miss, clear for a I-SLB miss
	 * We assume we aren't going to take any exceptions during this
	 * procedure.
	 */
	mflr	r10
#ifdef CONFIG_RELOCATABLE
	mtctr	r11
#endif

	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
	std	r10,PACA_EXSLB+EX_LR(r13)	/* save LR */
	std	r3,PACA_EXSLB+EX_DAR(r13)

	crset	4*cr0+eq
#ifdef CONFIG_PPC_STD_MMU_64
BEGIN_MMU_FTR_SECTION
	bl	slb_allocate_realmode
END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
#endif

	ld	r10,PACA_EXSLB+EX_LR(r13)
	ld	r3,PACA_EXSLB+EX_R3(r13)
	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
	mtlr	r10

	beq	8f		/* if bad address, make full stack frame */

	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
	beq-	2f

	/* All done -- return from exception. */

.machine	push
.machine	"power4"
	mtcrf	0x80,r9
	mtcrf	0x02,r9		/* I/D indication is in cr6 */
	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
.machine	pop

	RESTORE_PPR_PACA(PACA_EXSLB, r9)
	ld	r9,PACA_EXSLB+EX_R9(r13)
	ld	r10,PACA_EXSLB+EX_R10(r13)
	ld	r11,PACA_EXSLB+EX_R11(r13)
	ld	r12,PACA_EXSLB+EX_R12(r13)
	ld	r13,PACA_EXSLB+EX_R13(r13)
	rfid
	b	.	/* prevent speculative execution */

2:	mfspr	r11,SPRN_SRR0
	LOAD_HANDLER(r10,unrecov_slb)
	mtspr	SPRN_SRR0,r10
	ld	r10,PACAKMSR(r13)
	mtspr	SPRN_SRR1,r10
	rfid
	b	.

8:	mfspr	r11,SPRN_SRR0
	LOAD_HANDLER(r10,bad_addr_slb)
	mtspr	SPRN_SRR0,r10
	ld	r10,PACAKMSR(r13)
	mtspr	SPRN_SRR1,r10
	rfid
	b	.

EXC_COMMON_BEGIN(unrecov_slb)
	EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
	RECONCILE_IRQ_STATE(r10, r11)
	bl	save_nvgprs
1:	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	unrecoverable_exception
	b	1b

EXC_COMMON_BEGIN(bad_addr_slb)
	EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
	RECONCILE_IRQ_STATE(r10, r11)
	ld	r3, PACA_EXSLB+EX_DAR(r13)
	std	r3, _DAR(r1)
	beq	cr6, 2f
	li	r10, 0x480		/* fix trap number for I-SLB miss */
	std	r10, _TRAP(r1)
2:	bl	save_nvgprs
	addi	r3, r1, STACK_FRAME_OVERHEAD
	bl	slb_miss_bad_addr
	b	ret_from_except

714
EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x600)
715
716
	.globl hardware_interrupt_hv;
hardware_interrupt_hv:
717
	BEGIN_FTR_SECTION
718
		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
719
					    EXC_HV, SOFTEN_TEST_HV)
720
do_kvm_H0x500:
721
		KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
722
	FTR_SECTION_ELSE
723
		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
724
					    EXC_STD, SOFTEN_TEST_PR)
725
do_kvm_0x500:
726
		KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
727
	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
728
729
EXC_REAL_END(hardware_interrupt, 0x500, 0x600)

730
731
732
733
734
735
736
737
738
739
740
741
742
EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x4600)
	.globl hardware_interrupt_relon_hv;
hardware_interrupt_relon_hv:
	BEGIN_FTR_SECTION
		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV)
	FTR_SECTION_ELSE
		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_STD, SOFTEN_TEST_PR)
	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
EXC_VIRT_END(hardware_interrupt, 0x4500, 0x4600)

EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)


743
EXC_REAL(alignment, 0x600, 0x700)
744
EXC_VIRT(alignment, 0x4600, 0x4700, 0x600)
745
TRAMP_KVM(PACA_EXGEN, 0x600)
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
EXC_COMMON_BEGIN(alignment_common)
	mfspr	r10,SPRN_DAR
	std	r10,PACA_EXGEN+EX_DAR(r13)
	mfspr	r10,SPRN_DSISR
	stw	r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
	ld	r3,PACA_EXGEN+EX_DAR(r13)
	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
	std	r3,_DAR(r1)
	std	r4,_DSISR(r1)
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	alignment_exception
	b	ret_from_except

762
763

EXC_REAL(program_check, 0x700, 0x800)
764
EXC_VIRT(program_check, 0x4700, 0x4800, 0x700)
765
TRAMP_KVM(PACA_EXGEN, 0x700)
766
767
768
769
770
771
772
773
EXC_COMMON_BEGIN(program_check_common)
	EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	program_check_exception
	b	ret_from_except

774

775
EXC_REAL(fp_unavailable, 0x800, 0x900)
776
EXC_VIRT(fp_unavailable, 0x4800, 0x4900, 0x800)
777
TRAMP_KVM(PACA_EXGEN, 0x800)
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
EXC_COMMON_BEGIN(fp_unavailable_common)
	EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
	bne	1f			/* if from user, just load it up */
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	kernel_fp_unavailable_exception
	BUG_OPCODE
1:
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
BEGIN_FTR_SECTION
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif
	bl	load_up_fpu
	b	fast_exception_return
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	fp_unavailable_tm
	b	ret_from_except
#endif

807

808
EXC_REAL_MASKABLE(decrementer, 0x900, 0x980)
809
810
811
812
EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x4980, 0x900)
TRAMP_KVM(PACA_EXGEN, 0x900)
EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)

813

814
EXC_REAL_HV(hdecrementer, 0x980, 0xa00)
815
816
817
818
EXC_VIRT_HV(hdecrementer, 0x4980, 0x4a00, 0x980)
TRAMP_KVM_HV(PACA_EXGEN, 0x980)
EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)

819

820
EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0xb00)
821
EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x4b00, 0xa00)
822
TRAMP_KVM(PACA_EXGEN, 0xa00)
823
824
825
826
827
828
#ifdef CONFIG_PPC_DOORBELL
EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception)
#else
EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, unknown_exception)
#endif

829

830
EXC_REAL(trap_0b, 0xb00, 0xc00)
831
EXC_VIRT(trap_0b, 0x4b00, 0x4c00, 0xb00)
832
TRAMP_KVM(PACA_EXGEN, 0xb00)
833
834
EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)

835
836
#define LOAD_SYSCALL_HANDLER(reg)					\
	__LOAD_HANDLER(reg, system_call_common)
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886

/* Syscall routine is used twice, in reloc-off and reloc-on paths */
#define SYSCALL_PSERIES_1 					\
BEGIN_FTR_SECTION						\
	cmpdi	r0,0x1ebe ; 					\
	beq-	1f ;						\
END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\
	mr	r9,r13 ;					\
	GET_PACA(r13) ;						\
	mfspr	r11,SPRN_SRR0 ;					\
0:

#define SYSCALL_PSERIES_2_RFID 					\
	mfspr	r12,SPRN_SRR1 ;					\
	LOAD_SYSCALL_HANDLER(r10) ; 				\
	mtspr	SPRN_SRR0,r10 ; 				\
	ld	r10,PACAKMSR(r13) ;				\
	mtspr	SPRN_SRR1,r10 ; 				\
	rfid ; 							\
	b	. ;	/* prevent speculative execution */

#define SYSCALL_PSERIES_3					\
	/* Fast LE/BE switch system call */			\
1:	mfspr	r12,SPRN_SRR1 ;					\
	xori	r12,r12,MSR_LE ;				\
	mtspr	SPRN_SRR1,r12 ;					\
	rfid ;		/* return to userspace */		\
	b	. ;	/* prevent speculative execution */

#if defined(CONFIG_RELOCATABLE)
	/*
	 * We can't branch directly so we do it via the CTR which
	 * is volatile across system calls.
	 */
#define SYSCALL_PSERIES_2_DIRECT				\
	LOAD_SYSCALL_HANDLER(r12) ;				\
	mtctr	r12 ;						\
	mfspr	r12,SPRN_SRR1 ;					\
	li	r10,MSR_RI ;					\
	mtmsrd 	r10,1 ;						\
	bctr ;
#else
	/* We can branch directly */
#define SYSCALL_PSERIES_2_DIRECT				\
	mfspr	r12,SPRN_SRR1 ;					\
	li	r10,MSR_RI ;					\
	mtmsrd 	r10,1 ;			/* Set RI (EE=0) */	\
	b	system_call_common ;
#endif

887
EXC_REAL_BEGIN(system_call, 0xc00, 0xd00)
888
889
890
891
892
893
894
	 /*
	  * If CONFIG_KVM_BOOK3S_64_HANDLER is set, save the PPR (on systems
	  * that support it) before changing to HMT_MEDIUM. That allows the KVM
	  * code to save that value into the guest state (it is the guest's PPR
	  * value). Otherwise just change to HMT_MEDIUM as userspace has
	  * already saved the PPR.
	  */
895
896
897
898
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
	SET_SCRATCH0(r13)
	GET_PACA(r13)
	std	r9,PACA_EXGEN+EX_R9(r13)
899
900
	OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);
	HMT_MEDIUM;
901
	std	r10,PACA_EXGEN+EX_R10(r13)
902
	OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r9, CPU_FTR_HAS_PPR);
903
	mfcr	r9
904
	KVMTEST_PR(0xc00)
905
	GET_SCRATCH0(r13)
906
907
#else
	HMT_MEDIUM;
908
#endif
909
910
911
	SYSCALL_PSERIES_1
	SYSCALL_PSERIES_2_RFID
	SYSCALL_PSERIES_3
912
913
EXC_REAL_END(system_call, 0xc00, 0xd00)

914
915
916
917
918
919
920
EXC_VIRT_BEGIN(system_call, 0x4c00, 0x4d00)
	HMT_MEDIUM
	SYSCALL_PSERIES_1
	SYSCALL_PSERIES_2_DIRECT
	SYSCALL_PSERIES_3
EXC_VIRT_END(system_call, 0x4c00, 0x4d00)

921
922
TRAMP_KVM(PACA_EXGEN, 0xc00)

923

924
EXC_REAL(single_step, 0xd00, 0xe00)
925
EXC_VIRT(single_step, 0x4d00, 0x4e00, 0xd00)
926
TRAMP_KVM(PACA_EXGEN, 0xd00)
927
EXC_COMMON(single_step_common, 0xd00, single_step_exception)
928

929
EXC_REAL_OOL_HV(h_data_storage, 0xe00, 0xe20)
930
EXC_VIRT_NONE(0x4e00, 0x4e20)
931
932
933
934
935
936
937
938
939
940
941
942
943
TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00)
EXC_COMMON_BEGIN(h_data_storage_common)
	mfspr   r10,SPRN_HDAR
	std     r10,PACA_EXGEN+EX_DAR(r13)
	mfspr   r10,SPRN_HDSISR
	stw     r10,PACA_EXGEN+EX_DSISR(r13)
	EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
	bl      save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi    r3,r1,STACK_FRAME_OVERHEAD
	bl      unknown_exception
	b       ret_from_except

944

945
EXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0xe40)
946
EXC_VIRT_NONE(0x4e20, 0x4e40)
947
948
949
TRAMP_KVM_HV(PACA_EXGEN, 0xe20)
EXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception)

950

951
952
EXC_REAL_OOL_HV(emulation_assist, 0xe40, 0xe60)
EXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x4e60, 0xe40)
953
954
955
TRAMP_KVM_HV(PACA_EXGEN, 0xe40)
EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)

956

957
958
959
960
961
/*
 * hmi_exception trampoline is a special case. It jumps to hmi_exception_early
 * first, and then eventaully from there to the trampoline to get into virtual
 * mode.
 */
962
__EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0xe80, hmi_exception_early)
963
__TRAMP_REAL_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60)
964
EXC_VIRT_NONE(0x4e60, 0x4e80)
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
TRAMP_KVM_HV(PACA_EXGEN, 0xe60)
TRAMP_REAL_BEGIN(hmi_exception_early)
	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, 0xe60)
	mr	r10,r1			/* Save r1			*/
	ld	r1,PACAEMERGSP(r13)	/* Use emergency stack		*/
	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame		*/
	std	r9,_CCR(r1)		/* save CR in stackframe	*/
	mfspr	r11,SPRN_HSRR0		/* Save HSRR0 */
	std	r11,_NIP(r1)		/* save HSRR0 in stackframe	*/
	mfspr	r12,SPRN_HSRR1		/* Save SRR1 */
	std	r12,_MSR(r1)		/* save SRR1 in stackframe	*/
	std	r10,0(r1)		/* make stack chain pointer	*/
	std	r0,GPR0(r1)		/* save r0 in stackframe	*/
	std	r10,GPR1(r1)		/* save r1 in stackframe	*/
	EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
	EXCEPTION_PROLOG_COMMON_3(0xe60)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	hmi_exception_realmode
	/* Windup the stack. */
	/* Move original HSRR0 and HSRR1 into the respective regs */
	ld	r9,_MSR(r1)
	mtspr	SPRN_HSRR1,r9
	ld	r3,_NIP(r1)
	mtspr	SPRN_HSRR0,r3
	ld	r9,_CTR(r1)
	mtctr	r9
	ld	r9,_XER(r1)
	mtxer	r9
	ld	r9,_LINK(r1)
	mtlr	r9
	REST_GPR(0, r1)
	REST_8GPRS(2, r1)
	REST_GPR(10, r1)
	ld	r11,_CCR(r1)
	mtcr	r11
	REST_GPR(11, r1)
	REST_2GPRS(12, r1)
	/* restore original r1. */
	ld	r1,GPR1(r1)

	/*
	 * Go to virtual mode and pull the HMI event information from
	 * firmware.
	 */
	.globl hmi_exception_after_realmode
hmi_exception_after_realmode:
	SET_SCRATCH0(r13)
	EXCEPTION_PROLOG_0(PACA_EXGEN)
	b	tramp_real_hmi_exception

EXC_COMMON_ASYNC(hmi_exception_common, 0xe60, handle_hmi_exception)

1017

1018
1019
EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0xea0)
EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x4ea0, 0xe80)
1020
1021
1022
1023
1024
1025
1026
TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
#ifdef CONFIG_PPC_DOORBELL
EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
#else
EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
#endif

1027

1028
1029
EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0xec0)
EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x4ec0, 0xea0)
1030
1031
1032
TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)

1033

1034
EXC_REAL_NONE(0xec0, 0xf00)
1035
1036
EXC_VIRT_NONE(0x4ec0, 0x4f00)

1037

1038
1039
EXC_REAL_OOL(performance_monitor, 0xf00, 0xf20)
EXC_VIRT_OOL(performance_monitor, 0x4f00, 0x4f20, 0xf00)
1040
1041
1042
TRAMP_KVM(PACA_EXGEN, 0xf00)
EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception)

1043

1044
1045
EXC_REAL_OOL(altivec_unavailable, 0xf20, 0xf40)
EXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x4f40, 0xf20)
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
TRAMP_KVM(PACA_EXGEN, 0xf20)
EXC_COMMON_BEGIN(altivec_unavailable_common)
	EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	beq	1f
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  BEGIN_FTR_SECTION_NESTED(69)
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
#endif
	bl	load_up_altivec
	b	fast_exception_return
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	altivec_unavailable_tm
	b	ret_from_except
#endif
1:
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	altivec_unavailable_exception
	b	ret_from_except

1080

1081
1082
EXC_REAL_OOL(vsx_unavailable, 0xf40, 0xf60)
EXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x4f60, 0xf40)
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
TRAMP_KVM(PACA_EXGEN, 0xf40)
EXC_COMMON_BEGIN(vsx_unavailable_common)
	EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	beq	1f
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  BEGIN_FTR_SECTION_NESTED(69)
	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
	 * transaction), go do TM stuff
	 */
	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
	bne-	2f
  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
#endif
	b	load_up_vsx
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2:	/* User process was in a transaction */
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	vsx_unavailable_tm
	b	ret_from_except
#endif
1:
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	bl	save_nvgprs
	RECONCILE_IRQ_STATE(r10, r11)
	addi	r3,r1,STACK_FRAME_OVERHEAD
	bl	vsx_unavailable_exception
	b	ret_from_except

1116

1117
1118
EXC_REAL_OOL(facility_unavailable, 0xf60, 0xf80)
EXC_VIRT_OOL(facility_unavailable, 0x4f60, 0x4f80, 0xf60)
1119
1120
1121
TRAMP_KVM(PACA_EXGEN, 0xf60)
EXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception)

1122

1123
1124
EXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0xfa0)
EXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x4fa0, 0xf80)
1125
1126
1127
TRAMP_KVM_HV(PACA_EXGEN, 0xf80)
EXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception)

1128
1129

EXC_REAL_NONE(0xfa0, 0x1200)
1130
EXC_VIRT_NONE(0x4fa0, 0x5200)
1131

1132
#ifdef CONFIG_CBE_RAS
1133
EXC_REAL_HV(cbe_system_error, 0x1200, 0x1300)
1134
EXC_VIRT_NONE(0x5200, 0x5300)
1135
TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1200)
1136
EXC_COMMON(cbe_system_error_common, 0x1200, cbe_system_error_exception)
1137
1138
#else /* CONFIG_CBE_RAS */
EXC_REAL_NONE(0x1200, 0x1300)
1139
EXC_VIRT_NONE(0x5200, 0x5300)
1140
#endif
1141

1142

1143
EXC_REAL(instruction_breakpoint, 0x1300, 0x1400)
1144
EXC_VIRT(instruction_breakpoint, 0x5300, 0x5400, 0x1300)
1145
TRAMP_KVM_SKIP(PACA_EXGEN, 0x1300)
1146
1147
EXC_COMMON(instruction_breakpoint_common, 0x1300, instruction_breakpoint_exception)

1148
1149
EXC_REAL_NONE(0x1400, 0x1500)
EXC_VIRT_NONE(0x5400, 0x5500)
1150
1151

EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x1600)
1152
	mtspr	SPRN_SPRG_HSCRATCH0,r13
1153
	EXCEPTION_PROLOG_0(PACA_EXGEN)
1154
	EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
1155
1156
1157
1158

#ifdef CONFIG_PPC_DENORMALISATION
	mfspr	r10,SPRN_HSRR1
	mfspr	r11,SPRN_HSRR0		/* save HSRR0 */
1159
1160
1161
1162
	andis.	r10,r10,(HSRR1_DENORM)@h /* denorm? */
	addi	r11,r11,-4		/* HSRR0 is next instruction */
	bne+	denorm_assist
#endif
1163

1164
1165
1166
	KVMTEST_PR(0x1500)
	EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
EXC_REAL_END(denorm_exception_hv, 0x1500, 0x1600)
1167

1168
1169
1170
1171
1172
1173
#ifdef CONFIG_PPC_DENORMALISATION
EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x5600)
	b	exc_real_0x1500_denorm_exception_hv
EXC_VIRT_END(denorm_exception, 0x5500, 0x5600)
#else
EXC_VIRT_NONE(0x5500, 0x5600)
1174
1175
#endif

1176
TRAMP_KVM_SKIP(PACA_EXGEN, 0x1500)
1177

1178
#ifdef CONFIG_PPC_DENORMALISATION
1179
TRAMP_REAL_BEGIN(denorm_assist)
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
BEGIN_FTR_SECTION
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER6 do that here for all FP regs.
 */
	mfmsr	r10
	ori	r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
	xori	r10,r10,(MSR_FE0|MSR_FE1)
	mtmsrd	r10
	sync
1190
1191
1192
1193
1194
1195
1196
1197

#define FMR2(n)  fmr (n), (n) ; fmr n+1, n+1
#define FMR4(n)  FMR2(n) ; FMR2(n+2)
#define FMR8(n)  FMR4(n) ; FMR4(n+4)
#define FMR16(n) FMR8(n) ; FMR8(n+8)
#define FMR32(n) FMR16(n) ; FMR16(n+16)
	FMR32(0)

1198
1199
1200
1201
1202
1203
1204
1205
1206
FTR_SECTION_ELSE
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER7 do that here for the first 32 VSX registers only.
 */
	mfmsr	r10
	oris	r10,r10,MSR_VSX@h
	mtmsrd	r10
	sync
1207
1208
1209
1210
1211
1212
1213
1214

#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
	XVCPSGNDP32(0)

1215
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225

BEGIN_FTR_SECTION
	b	denorm_done
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
/*
 * To denormalise we need to move a copy of the register to itself.
 * For POWER8 we need to do that for all 64 VSX registers
 */
	XVCPSGNDP32(32)
denorm_done:
1226
1227
1228
	mtspr	SPRN_HSRR0,r11
	mtcrf	0x80,r9
	ld	r9,PACA_EXGEN+EX_R9(r13)
1229
	RESTORE_PPR_PACA(PACA_EXGEN, r10)
1230
1231
1232
1233
BEGIN_FTR_SECTION
	ld	r10,PACA_EXGEN+EX_CFAR(r13)
	mtspr	SPRN_CFAR,r10
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1234
1235
1236
1237
1238
1239
1240
1241
	ld	r10,PACA_EXGEN+EX_R10(r13)
	ld	r11,PACA_EXGEN+EX_R11(r13)
	ld	r12,PACA_EXGEN+EX_R12(r13)
	ld	r13,PACA_EXGEN+EX_R13(r13)
	HRFID
	b	.
#endif

1242
1243
1244
1245
1246
EXC_COMMON_HV(denorm_common, 0x1500, unknown_exception)


#ifdef CONFIG_CBE_RAS
EXC_REAL_HV(cbe_maintenance, 0x1600, 0x1700)
1247
EXC_VIRT_NONE(0x5600, 0x5700)
1248
TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1600)
1249
EXC_COMMON(cbe_maintenance_common, 0x1600, cbe_maintenance_exception)
1250
1251
#else /* CONFIG_CBE_RAS */
EXC_REAL_NONE(0x1600, 0x1700)
1252
EXC_VIRT_NONE(0x5600, 0x5700)
1253
1254
#endif

1255

1256
EXC_REAL(altivec_assist, 0x1700, 0x1800)
1257
EXC_VIRT(altivec_assist, 0x5700, 0x5800, 0x1700)
1258
TRAMP_KVM(PACA_EXGEN, 0x1700)
1259
1260
1261
1262
1263
1264
#ifdef CONFIG_ALTIVEC
EXC_COMMON(altivec_assist_common, 0x1700, altivec_assist_exception)
#else
EXC_COMMON(altivec_assist_common, 0x1700, unknown_exception)
#endif

1265
1266
1267

#ifdef CONFIG_CBE_RAS
EXC_REAL_HV(cbe_thermal, 0x1800, 0x1900)
1268
EXC_VIRT_NONE(0x5800, 0x5900)
1269
TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1800)
1270
EXC_COMMON(cbe_thermal_common, 0x1800, cbe_thermal_exception)
1271
1272
#else /* CONFIG_CBE_RAS */
EXC_REAL_NONE(0x1800, 0x1900)
1273
EXC_VIRT_NONE(0x5800, 0x5900)
1274
1275
1276
#endif


1277
/*
1278
1279
1280
1281
 * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
 * - If it was a decrementer interrupt, we bump the dec to max and and return.
 * - If it was a doorbell we return immediately since doorbells are edge
 *   triggered and won't automatically refire.
1282
1283
 * - If it was a HMI we return immediately since we handled it in realmode
 *   and it won't refire.
1284
1285
 * - else we hard disable and return.
 * This is called with r10 containing the value to OR to the paca field.
1286
 */
1287
1288
1289
1290
1291
1292
#define MASKED_INTERRUPT(_H)				\
masked_##_H##interrupt:					\
	std	r11,PACA_EXGEN+EX_R11(r13);		\
	lbz	r11,PACAIRQHAPPENED(r13);		\
	or	r11,r11,r10;				\
	stb	r11,PACAIRQHAPPENED(r13);		\
1293
1294
	cmpwi	r10,PACA_IRQ_DEC;			\
	bne	1f;					\
1295
1296
1297
1298
	lis	r10,0x7fff;				\
	ori	r10,r10,0xffff;				\
	mtspr	SPRN_DEC,r10;				\
	b	2f;					\
1299
1:	cmpwi	r10,PACA_IRQ_DBELL;			\
1300
1301
	beq	2f;					\
	cmpwi	r10,PACA_IRQ_HMI;			\
1302
1303
	beq	2f;					\
	mfspr	r10,SPRN_##_H##SRR1;			\
1304
1305
1306
1307
1308
1309
1310
1311
1312
	rldicl	r10,r10,48,1; /* clear MSR_EE */	\
	rotldi	r10,r10,16;				\
	mtspr	SPRN_##_H##SRR1,r10;			\
2:	mtcrf	0x80,r9;				\
	ld	r9,PACA_EXGEN+EX_R9(r13);		\
	ld	r10,PACA_EXGEN+EX_R10(r13);		\
	ld	r11,PACA_EXGEN+EX_R11(r13);		\
	GET_SCRATCH0(r13);				\
	##_H##rfid;					\
1313
	b	.
1314
1315
1316
1317
1318
1319
1320

/*
 * Real mode exceptions actually use this too, but alternate
 * instruction code patches (which end up in the common .text area)
 * cannot reach these if they are put there.
 */
USE_FIXED_SECTION(virt_trampolines)
1321
1322
	MASKED_INTERRUPT()
	MASKED_INTERRUPT(H)
1323

1324
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1325
TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
	/*
	 * Here all GPRs are unchanged from when the interrupt happened
	 * except for r13, which is saved in SPRG_SCRATCH0.
	 */
	mfspr	r13, SPRN_SRR0
	addi	r13, r13, 4
	mtspr	SPRN_SRR0, r13
	GET_SCRATCH0(r13)
	rfid
	b	.

1337
TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
	/*
	 * Here all GPRs are unchanged from when the interrupt happened
	 * except for r13, which is saved in SPRG_SCRATCH0.
	 */
	mfspr	r13, SPRN_HSRR0
	addi	r13, r13, 4
	mtspr	SPRN_HSRR0, r13
	GET_SCRATCH0(r13)
	hrfid
	b	.
#endif

1350
/*
1351
1352
1353
1354
 * Ensure that any handlers that get invoked from the exception prologs
 * above are below the first 64KB (0x10000) of the kernel image because
 * the prologs assemble the addresses of these handlers using the
 * LOAD_HANDLER macro, which uses an ori instruction.
1355
1356
1357
1358
1359
 */

/*** Common interrupt handlers ***/


1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
	/*
	 * Relocation-on interrupts: A subset of the interrupts can be delivered
	 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
	 * it.  Addresses are the same as the original interrupt addresses, but
	 * offset by 0xc000000000004000.
	 * It's impossible to receive interrupts below 0x300 via this mechanism.
	 * KVM: None of these traps are from the guest ; anything that escalated
	 * to HV=1 from HV=0 is delivered via real mode handlers.
	 */

	/*
	 * This uses the standard macro, since the original 0x300 vector
	 * only has extra guff for STAB-based processors -- which never
	 * come here.
	 */
1375

1376
EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
1377
	b	__ppc64_runlatch_on
1378

1379
USE_FIXED_SECTION(virt_trampolines)
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
	/*
	 * The __end_interrupts marker must be past the out-of-line (OOL)
	 * handlers, so that they are copied to real address 0x100 when running
	 * a relocatable kernel. This ensures they can be reached from the short
	 * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
	 * directly, without using LOAD_HANDLER().
	 */
	.align	7
	.globl	__end_interrupts
__end_interrupts:
1390
DEFINE_FIXED_SYMBOL(__end_interrupts)