entry.S 19 KB
Newer Older
Catalin Marinas's avatar
Catalin Marinas committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative-asm.h>
Catalin Marinas's avatar
Catalin Marinas committed
25
26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
Catalin Marinas's avatar
Catalin Marinas committed
28
#include <asm/errno.h>
29
#include <asm/esr.h>
Catalin Marinas's avatar
Catalin Marinas committed
30
31
32
#include <asm/thread_info.h>
#include <asm/unistd.h>

Larry Bassel's avatar
Larry Bassel committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

59
60
61
62
63
64
65
#ifdef CONFIG_IPIPE
#define PREEMPT_SCHEDULE_IRQ	__ipipe_preempt_schedule_irq
#else /* !CONFIG_IPIPE */
#ifdef ret_from_exception	ret_to_user
#define PREEMPT_SCHEDULE_IRQ	preempt_schedule_irq
#endif /* CONFIG_IPIPE */

Catalin Marinas's avatar
Catalin Marinas committed
66
67
68
69
70
71
72
73
74
75
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

	.macro	kernel_entry, el, regsize = 64
76
	sub	sp, sp, #S_FRAME_SIZE
Catalin Marinas's avatar
Catalin Marinas committed
77
78
79
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

Catalin Marinas's avatar
Catalin Marinas committed
96
97
	.if	\el == 0
	mrs	x21, sp_el0
98
99
100
	get_thread_info tsk			// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug
	disable_step_tsk x19, x20		// exceptions when scheduling.
Catalin Marinas's avatar
Catalin Marinas committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
	.else
	add	x21, sp, #S_FRAME_SIZE
	.endif
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
	stp	x22, x23, [sp, #S_PC]

	/*
	 * Set syscallno to -1 by default (overridden later if real syscall).
	 */
	.if	\el == 0
	mvn	x21, xzr
	str	x21, [sp, #S_SYSCALLNO]
	.endif

	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

	.macro	kernel_exit, el, ret = 0
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
Larry Bassel's avatar
Larry Bassel committed
129
	ct_user_enter
Catalin Marinas's avatar
Catalin Marinas committed
130
	ldr	x23, [sp, #S_SP]		// load return stack pointer
131
	msr	sp_el0, x23
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

#ifdef CONFIG_ARM64_ERRATUM_845719
	alternative_insn						\
	"nop",								\
	"tbz x22, #4, 1f",						\
	ARM64_WORKAROUND_845719
#ifdef CONFIG_PID_IN_CONTEXTIDR
	alternative_insn						\
	"nop; nop",							\
	"mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:",		\
	ARM64_WORKAROUND_845719
#else
	alternative_insn						\
	"nop",								\
	"msr contextidr_el1, xzr; 1:",					\
	ARM64_WORKAROUND_845719
#endif
#endif
Catalin Marinas's avatar
Catalin Marinas committed
150
	.endif
151
152
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
Catalin Marinas's avatar
Catalin Marinas committed
153
154
155
	.if	\ret
	ldr	x1, [sp, #S_X1]			// preserve x0 (syscall return)
	.else
156
	ldp	x0, x1, [sp, #16 * 0]
Catalin Marinas's avatar
Catalin Marinas committed
157
	.endif
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
Catalin Marinas's avatar
Catalin Marinas committed
174
175
176
177
178
	eret					// return to kernel
	.endm

	.macro	get_thread_info, rd
	mov	\rd, sp
179
	and	\rd, \rd, #~(THREAD_SIZE - 1)	// top of stack
Catalin Marinas's avatar
Catalin Marinas committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
	.endm

/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
sc_nr	.req	x25		// number of system calls
scno	.req	x26		// syscall number
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
197
198
	adrp	x1, handle_arch_irq
	ldr	x1, [x1, #:lo12:handle_arch_irq]
199
200
	mov	x0, sp
	blr	x1
201
202
203
204
#ifdef CONFIG_IPIPE
	bl  __ipipe_check_root_interruptible
	cmp x0, #1
#endif /* CONFIG_IPIPE */
Catalin Marinas's avatar
Catalin Marinas committed
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
	.endm

	.text

/*
 * Exception vectors.
 */

	.align	11
ENTRY(vectors)
	ventry	el1_sync_invalid		// Synchronous EL1t
	ventry	el1_irq_invalid			// IRQ EL1t
	ventry	el1_fiq_invalid			// FIQ EL1t
	ventry	el1_error_invalid		// Error EL1t

	ventry	el1_sync			// Synchronous EL1h
	ventry	el1_irq				// IRQ EL1h
	ventry	el1_fiq_invalid			// FIQ EL1h
	ventry	el1_error_invalid		// Error EL1h

	ventry	el0_sync			// Synchronous 64-bit EL0
	ventry	el0_irq				// IRQ 64-bit EL0
	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	ventry	el0_error_invalid		// Error 64-bit EL0

#ifdef CONFIG_COMPAT
	ventry	el0_sync_compat			// Synchronous 32-bit EL0
	ventry	el0_irq_compat			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	ventry	el0_error_invalid_compat	// Error 32-bit EL0
#else
	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	ventry	el0_irq_invalid			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	ventry	el0_error_invalid		// Error 32-bit EL0
#endif
END(vectors)

/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
	kernel_entry el, \regsize
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
	b	bad_mode
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
303
304
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
Catalin Marinas's avatar
Catalin Marinas committed
305
	b.eq	el1_da
Mark Rutland's avatar
Mark Rutland committed
306
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
Catalin Marinas's avatar
Catalin Marinas committed
307
	b.eq	el1_undef
Mark Rutland's avatar
Mark Rutland committed
308
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
309
	b.eq	el1_sp_pc
Mark Rutland's avatar
Mark Rutland committed
310
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
311
	b.eq	el1_sp_pc
Mark Rutland's avatar
Mark Rutland committed
312
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
313
	b.eq	el1_undef
Mark Rutland's avatar
Mark Rutland committed
314
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
315
316
317
318
319
320
321
	b.ge	el1_dbg
	b	el1_inv
el1_da:
	/*
	 * Data abort handling
	 */
	mrs	x0, far_el1
322
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
	// re-enable interrupts if they were enabled in the aborted context
	tbnz	x23, #7, 1f			// PSR_I_BIT
	enable_irq
1:
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	// disable interrupts before pulling preserved data off the stack
	disable_irq
	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
338
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
339
340
341
342
343
344
	mov	x2, sp
	b	do_sp_pc_abort
el1_undef:
	/*
	 * Undefined instruction
	 */
345
#ifdef CONFIG_IPIPE
346
347
348
349
350
	mov	x0, #7				//@ x0 = IPIPE_TRAP_UNDEFINSTR
	mov	x1, sp				//@ x1 = &regs
	bl	__ipipe_notify_trap		//@ branch to trap handler
	cmp	x0, #0
	bne	__ipipe_fast_svc_irq_exit
351
#endif /* CONFIG_IPIPE */
352
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
353
354
355
356
357
358
	mov	x0, sp
	b	do_undefinstr
el1_dbg:
	/*
	 * Debug exception handling
	 */
Mark Rutland's avatar
Mark Rutland committed
359
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
360
	cinc	x24, x24, eq			// set bit '0'
Catalin Marinas's avatar
Catalin Marinas committed
361
362
363
364
365
366
367
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
368
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
369
370
371
372
373
374
375
376
377
	mov	x0, sp
	mov	x1, #BAD_SYNC
	mrs	x2, esr_el1
	b	bad_mode
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
378
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
379
380
381
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
382

Catalin Marinas's avatar
Catalin Marinas committed
383
	irq_handler
384

385
386
387
#ifdef CONFIG_IPIPE
	bne	__ipipe_fast_svc_irq_exit
#endif
Catalin Marinas's avatar
Catalin Marinas committed
388
#ifdef CONFIG_PREEMPT
389
	get_thread_info tsk
Neil Zhang's avatar
Neil Zhang committed
390
	ldr	w24, [tsk, #TI_PREEMPT]		// get preempt count
391
	cbnz	w24, 1f				// preempt count != 0
Catalin Marinas's avatar
Catalin Marinas committed
392
393
394
395
396
	ldr	x0, [tsk, #TI_FLAGS]		// get flags
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
397
398
399
#ifdef CONFIG_IPIPE
__ipipe_fast_svc_irq_exit:
#endif
Catalin Marinas's avatar
Catalin Marinas committed
400
401
402
403
404
405
406
407
408
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
409
1:	bl	PREEMPT_SCHEDULE_IRQ
Catalin Marinas's avatar
Catalin Marinas committed
410
411
412
413
414
415
416
417
418
419
420
421
	ldr	x0, [tsk, #TI_FLAGS]		// get new tasks TI_FLAGS
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
422
423
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
Catalin Marinas's avatar
Catalin Marinas committed
424
	b.eq	el0_svc
Mark Rutland's avatar
Mark Rutland committed
425
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
426
	b.eq	el0_da
Mark Rutland's avatar
Mark Rutland committed
427
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
428
	b.eq	el0_ia
Mark Rutland's avatar
Mark Rutland committed
429
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
430
	b.eq	el0_fpsimd_acc
Mark Rutland's avatar
Mark Rutland committed
431
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
432
	b.eq	el0_fpsimd_exc
Mark Rutland's avatar
Mark Rutland committed
433
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
Catalin Marinas's avatar
Catalin Marinas committed
434
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
435
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
436
	b.eq	el0_sp_pc
Mark Rutland's avatar
Mark Rutland committed
437
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
438
	b.eq	el0_sp_pc
Mark Rutland's avatar
Mark Rutland committed
439
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
440
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
441
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
442
443
444
445
446
447
448
449
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
450
451
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
Catalin Marinas's avatar
Catalin Marinas committed
452
	b.eq	el0_svc_compat
Mark Rutland's avatar
Mark Rutland committed
453
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
454
	b.eq	el0_da
Mark Rutland's avatar
Mark Rutland committed
455
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
456
	b.eq	el0_ia
Mark Rutland's avatar
Mark Rutland committed
457
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
458
	b.eq	el0_fpsimd_acc
Mark Rutland's avatar
Mark Rutland committed
459
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
460
	b.eq	el0_fpsimd_exc
Mark Rutland's avatar
Mark Rutland committed
461
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
462
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
463
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
464
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
465
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
466
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
467
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
468
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
469
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
470
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
471
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
472
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
473
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
474
475
476
477
478
479
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
480
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
Catalin Marinas's avatar
Catalin Marinas committed
481
482
483
484
485
486
487
488
489
490
491
492
493
494
	uxtw	scno, w7			// syscall number in w7 (r7)
	mov     sc_nr, #__NR_compat_syscalls
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
495
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
496
	// enable interrupts before calling the main handler
497
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
498
	ct_user_exit
499
	bic	x0, x26, #(0xff << 56)
Catalin Marinas's avatar
Catalin Marinas committed
500
501
	mov	x1, x25
	mov	x2, sp
502
	bl	do_mem_abort
503
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
504
505
506
507
el0_ia:
	/*
	 * Instruction abort handling
	 */
508
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
509
	// enable interrupts before calling the main handler
510
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
511
	ct_user_exit
512
	mov	x0, x26
Catalin Marinas's avatar
Catalin Marinas committed
513
514
	orr	x1, x25, #1 << 24		// use reserved ISS bit for instruction aborts
	mov	x2, sp
515
	bl	do_mem_abort
516
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
517
518
519
520
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
521
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
522
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
523
524
	mov	x0, x25
	mov	x1, sp
525
	bl	do_fpsimd_acc
526
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
527
528
529
530
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
531
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
532
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
533
534
	mov	x0, x25
	mov	x1, sp
535
	bl	do_fpsimd_exc
536
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
537
538
539
540
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
541
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
542
	// enable interrupts before calling the main handler
543
	enable_dbg_and_irq
544
	ct_user_exit
545
	mov	x0, x26
Catalin Marinas's avatar
Catalin Marinas committed
546
547
	mov	x1, x25
	mov	x2, sp
548
	bl	do_sp_pc_abort
549
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
550
551
552
553
el0_undef:
	/*
	 * Undefined instruction
	 */
554
#ifdef CONFIG_IPIPE
555
556
557
558
559
	mov	x0, #7				// x0 = IPIPE_TRAP_UNDEFINSTR
	mov	x1, sp				// x1 = &regs
	bl	__ipipe_notify_trap		// branch to trap handler
	cmp	x0, #0
	bne	ret_from_exception
560
#endif /* CONFIG_IPIPE */
561
	// enable interrupts before calling the main handler
562
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
563
	ct_user_exit
564
	mov	x0, sp
565
	bl	do_undefinstr
566
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
567
568
569
570
571
572
573
574
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
575
576
	bl	do_debug_exception
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
577
	ct_user_exit
578
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
579
el0_inv:
580
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
581
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
582
583
584
	mov	x0, sp
	mov	x1, #BAD_SYNC
	mrs	x2, esr_el1
585
	bl	bad_mode
586
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
587
588
589
590
591
592
593
594
595
596
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
597

Larry Bassel's avatar
Larry Bassel committed
598
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
599
	irq_handler
600

601
602
603
#ifdef CONFIG_IPIPE
	bne	__ipipe_ret_to_user_irqs_disabled
#endif	/* CONFIG_IPIPE */
Catalin Marinas's avatar
Catalin Marinas committed
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
	add	x8, x0, #THREAD_CPU_CONTEXT
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
	add	x8, x1, #THREAD_CPU_CONTEXT
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
	ret
ENDPROC(cpu_switch_to)

/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
	disable_irq				// disable interrupts
	ldr	x1, [tsk, #TI_FLAGS]
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, fast_work_pending
649
	enable_step_tsk x1, x2
Catalin Marinas's avatar
Catalin Marinas committed
650
651
	kernel_exit 0, ret = 1

652
#ifdef CONFIG_IPIPE
653
654
655
656
ret_from_exception:
	ldr	x0, [tsk, #TI_IPIPE]
	tst	x0, #_TIP_HEAD
	b.eq	ret_to_user
657
__ipipe_ret_to_user:
658
659
	disable_irq
__ipipe_ret_to_user_irqs_disabled:
660
661
662
	kernel_exit 0, ret = 0
#endif /* CONFIG_IPIPE */

Catalin Marinas's avatar
Catalin Marinas committed
663
664
665
666
667
668
669
/*
 * Ok, we need to do extra processing, enter the slow path.
 */
fast_work_pending:
	str	x0, [sp, #S_X0]			// returned x0
work_pending:
	tbnz	x1, #TIF_NEED_RESCHED, work_resched
670
	/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
Catalin Marinas's avatar
Catalin Marinas committed
671
672
673
674
	ldr	x2, [sp, #S_PSTATE]
	mov	x0, sp				// 'regs'
	tst	x2, #PSR_MODE_MASK		// user mode regs?
	b.ne	no_work_pending			// returning to kernel
675
	enable_irq				// enable interrupts for do_notify_resume()
Catalin Marinas's avatar
Catalin Marinas committed
676
677
678
	bl	do_notify_resume
	b	ret_to_user
work_resched:
679
680
681
682
683
684
685
686
687
688
#ifdef CONFIG_IPIPE
	/*
	In armv7, ipipe inserts the following (in do_work_pending, NEED_RESCHED case):
	local_irq_disable();
	hard_cond_local_irq_enable();
	*/
	bl ipipe_stall_root //local_irq_disable() -> { ipipe_stall_root(); barrier(); }
	dmb sy				//barrier(); -> __memory_barrier();
	msr daifclr, #2 //hard_cond_local_irq_enable();
#endif
Catalin Marinas's avatar
Catalin Marinas committed
689
690
691
692
693
	bl	schedule

/*
 * "slow" syscall return path.
 */
694
ret_to_user:
Catalin Marinas's avatar
Catalin Marinas committed
695
696
697
698
	disable_irq				// disable interrupts
	ldr	x1, [tsk, #TI_FLAGS]
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
699
	enable_step_tsk x1, x2
Catalin Marinas's avatar
Catalin Marinas committed
700
701
702
703
704
705
706
707
708
no_work_pending:
	kernel_exit 0, ret = 0
ENDPROC(ret_to_user)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
709
710
711
712
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
Catalin Marinas's avatar
Catalin Marinas committed
713
714
715
716
717
718
719
720
721
722
723
724
725
	b	ret_to_user
ENDPROC(ret_from_fork)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
	uxtw	scno, w8			// syscall number in w8
	mov	sc_nr, #__NR_syscalls
el0_svc_naked:					// compat entry point
	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
726
	enable_dbg_and_irq
727
728
729
730
731
732
733
734
735

#ifdef CONFIG_IPIPE
	mov	x1, sp
	mov	x0, scno
	bl	__ipipe_syscall_root
	cmp	x0, #0
	blt	__ipipe_ret_to_user
	get_thread_info tsk
	bgt	ret_to_user
736
737
738
739
740
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
741
	ldp	x2, x3, [sp, #S_X2]
742
743
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
744

745
746
#endif /* CONFIG_IPIPE */

Larry Bassel's avatar
Larry Bassel committed
747
	ct_user_exit 1
Catalin Marinas's avatar
Catalin Marinas committed
748

749
750
751
	ldr	x16, [tsk, #TI_FLAGS]		// check for syscall hooks
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
752
753
754
	cmp     scno, sc_nr                     // check upper syscall limit
	b.hs	ni_sys
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
755
756
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
757
758
ni_sys:
	mov	x0, sp
759
760
	bl	do_ni_syscall
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
761
762
763
764
765
766
767
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
768
769
770
771
772
773
	mov	w0, #-1				// set default errno for
	cmp     scno, x0			// user-issued syscall(-1)
	b.ne	1f
	mov	x0, #-ENOSYS
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
774
	bl	syscall_trace_enter
775
776
	cmp	w0, #-1				// skip the syscall?
	b.eq	__sys_trace_return_skipped
Catalin Marinas's avatar
Catalin Marinas committed
777
778
779
	uxtw	scno, w0			// syscall number (possibly new)
	mov	x1, sp				// pointer to regs
	cmp	scno, sc_nr			// check upper syscall limit
780
	b.hs	__ni_sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
781
782
783
784
785
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
786
	blr	x16				// call sys_* routine
Catalin Marinas's avatar
Catalin Marinas committed
787
788

__sys_trace_return:
789
790
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
791
792
	mov	x0, sp
	bl	syscall_trace_exit
Catalin Marinas's avatar
Catalin Marinas committed
793
794
	b	ret_to_user

795
796
797
798
799
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

Catalin Marinas's avatar
Catalin Marinas committed
800
801
802
803
804
805
806
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842

/*
 * I-pipe tracer helper functions
 */
#if defined(CONFIG_FRAME_POINTER) && (CONFIG_IPIPE_TRACE)
	.text
	.align 3
	.type arm64_return_addr %function
	.global arm64_return_addr

arm64_return_addr:
	mov	x16, x0
	ldr	x0, [x29]
3:
	cmp	x0, #0
	beq	1f				// frame list hit end, bail
	cmp	x16, #0
	beq	2f				// reached desired frame
	ldr	x0, [x0]		// else continue, get next fp
	sub	x16, x16, #1
	b	3b
2:
	add	x0, x0, #8		// get target return address (LR)
1:
	ret
ENDPROC(arm64_return_addr)
#endif
#ifdef CONFIG_IPIPE_TRACE_MCOUNT
	.text
	.align 3
	.type mcount %function
	.global mcount
mcount:
	ret		// just return
ENDPROC(mcount)
#endif