entry.S 18.9 KB
Newer Older
Catalin Marinas's avatar
Catalin Marinas committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative-asm.h>
Catalin Marinas's avatar
Catalin Marinas committed
25
26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
Catalin Marinas's avatar
Catalin Marinas committed
28
#include <asm/errno.h>
29
#include <asm/esr.h>
Catalin Marinas's avatar
Catalin Marinas committed
30
31
32
#include <asm/thread_info.h>
#include <asm/unistd.h>

Larry Bassel's avatar
Larry Bassel committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

59
60
61
#ifdef CONFIG_IPIPE
#define PREEMPT_SCHEDULE_IRQ	__ipipe_preempt_schedule_irq
#else /* !CONFIG_IPIPE */
62
#define ret_from_exception	ret_to_user
63
64
65
#define PREEMPT_SCHEDULE_IRQ	preempt_schedule_irq
#endif /* CONFIG_IPIPE */

66
67
68
69
70
71
72
73
#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_IPIPE)
#define TRACE_IRQSON  	bl  trace_hardirqs_on
#define TRACE_IRQSOFF  	bl  trace_hardirqs_off
#else
#define TRACE_IRQSON
#define TRACE_IRQSOFF
#endif

Catalin Marinas's avatar
Catalin Marinas committed
74
75
76
77
78
79
80
81
82
83
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

	.macro	kernel_entry, el, regsize = 64
84
	sub	sp, sp, #S_FRAME_SIZE
Catalin Marinas's avatar
Catalin Marinas committed
85
86
87
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

Catalin Marinas's avatar
Catalin Marinas committed
104
105
	.if	\el == 0
	mrs	x21, sp_el0
106
107
108
	get_thread_info tsk			// Ensure MDSCR_EL1.SS is clear,
	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug
	disable_step_tsk x19, x20		// exceptions when scheduling.
Catalin Marinas's avatar
Catalin Marinas committed
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
	.else
	add	x21, sp, #S_FRAME_SIZE
	.endif
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
	stp	x22, x23, [sp, #S_PC]

	/*
	 * Set syscallno to -1 by default (overridden later if real syscall).
	 */
	.if	\el == 0
	mvn	x21, xzr
	str	x21, [sp, #S_SYSCALLNO]
	.endif

	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

	.macro	kernel_exit, el, ret = 0
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
Larry Bassel's avatar
Larry Bassel committed
137
	ct_user_enter
Catalin Marinas's avatar
Catalin Marinas committed
138
	ldr	x23, [sp, #S_SP]		// load return stack pointer
139
	msr	sp_el0, x23
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

#ifdef CONFIG_ARM64_ERRATUM_845719
	alternative_insn						\
	"nop",								\
	"tbz x22, #4, 1f",						\
	ARM64_WORKAROUND_845719
#ifdef CONFIG_PID_IN_CONTEXTIDR
	alternative_insn						\
	"nop; nop",							\
	"mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:",		\
	ARM64_WORKAROUND_845719
#else
	alternative_insn						\
	"nop",								\
	"msr contextidr_el1, xzr; 1:",					\
	ARM64_WORKAROUND_845719
#endif
#endif
Catalin Marinas's avatar
Catalin Marinas committed
158
	.endif
159
160
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
Catalin Marinas's avatar
Catalin Marinas committed
161
162
163
	.if	\ret
	ldr	x1, [sp, #S_X1]			// preserve x0 (syscall return)
	.else
164
	ldp	x0, x1, [sp, #16 * 0]
Catalin Marinas's avatar
Catalin Marinas committed
165
	.endif
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
Catalin Marinas's avatar
Catalin Marinas committed
182
183
184
185
186
	eret					// return to kernel
	.endm

	.macro	get_thread_info, rd
	mov	\rd, sp
187
	and	\rd, \rd, #~(THREAD_SIZE - 1)	// top of stack
Catalin Marinas's avatar
Catalin Marinas committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
	.endm

/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
sc_nr	.req	x25		// number of system calls
scno	.req	x26		// syscall number
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
205
206
	adrp	x1, handle_arch_irq
	ldr	x1, [x1, #:lo12:handle_arch_irq]
207
208
	mov	x0, sp
	blr	x1
209
210
#ifdef CONFIG_IPIPE
	bl  __ipipe_check_root_interruptible
211
	cmp w0, #1
212
#endif /* CONFIG_IPIPE */
Catalin Marinas's avatar
Catalin Marinas committed
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
	.endm

	.text

/*
 * Exception vectors.
 */

	.align	11
ENTRY(vectors)
	ventry	el1_sync_invalid		// Synchronous EL1t
	ventry	el1_irq_invalid			// IRQ EL1t
	ventry	el1_fiq_invalid			// FIQ EL1t
	ventry	el1_error_invalid		// Error EL1t

	ventry	el1_sync			// Synchronous EL1h
	ventry	el1_irq				// IRQ EL1h
	ventry	el1_fiq_invalid			// FIQ EL1h
	ventry	el1_error_invalid		// Error EL1h

	ventry	el0_sync			// Synchronous 64-bit EL0
	ventry	el0_irq				// IRQ 64-bit EL0
	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	ventry	el0_error_invalid		// Error 64-bit EL0

#ifdef CONFIG_COMPAT
	ventry	el0_sync_compat			// Synchronous 32-bit EL0
	ventry	el0_irq_compat			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	ventry	el0_error_invalid_compat	// Error 32-bit EL0
#else
	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	ventry	el0_irq_invalid			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	ventry	el0_error_invalid		// Error 32-bit EL0
#endif
END(vectors)

/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
	kernel_entry el, \regsize
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
	b	bad_mode
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
311
312
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
Catalin Marinas's avatar
Catalin Marinas committed
313
	b.eq	el1_da
Mark Rutland's avatar
Mark Rutland committed
314
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
Catalin Marinas's avatar
Catalin Marinas committed
315
	b.eq	el1_undef
Mark Rutland's avatar
Mark Rutland committed
316
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
317
	b.eq	el1_sp_pc
Mark Rutland's avatar
Mark Rutland committed
318
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
319
	b.eq	el1_sp_pc
Mark Rutland's avatar
Mark Rutland committed
320
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
321
	b.eq	el1_undef
Mark Rutland's avatar
Mark Rutland committed
322
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
323
324
325
326
327
328
329
	b.ge	el1_dbg
	b	el1_inv
el1_da:
	/*
	 * Data abort handling
	 */
	mrs	x0, far_el1
330
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
	// re-enable interrupts if they were enabled in the aborted context
	tbnz	x23, #7, 1f			// PSR_I_BIT
	enable_irq
1:
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	// disable interrupts before pulling preserved data off the stack
	disable_irq
	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
346
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
347
348
349
350
351
352
	mov	x2, sp
	b	do_sp_pc_abort
el1_undef:
	/*
	 * Undefined instruction
	 */
353
#ifdef CONFIG_IPIPE
354
355
356
	mov	x0, #7				//@ x0 = IPIPE_TRAP_UNDEFINSTR
	mov	x1, sp				//@ x1 = &regs
	bl	__ipipe_notify_trap		//@ branch to trap handler
357
	cmp	w0, #0
358
	bne	ipipe_fast_svc_irq_exit
359
#endif /* CONFIG_IPIPE */
360
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
361
362
363
364
365
366
	mov	x0, sp
	b	do_undefinstr
el1_dbg:
	/*
	 * Debug exception handling
	 */
Mark Rutland's avatar
Mark Rutland committed
367
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
368
	cinc	x24, x24, eq			// set bit '0'
Catalin Marinas's avatar
Catalin Marinas committed
369
370
371
372
373
374
375
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
376
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
377
378
379
380
381
382
383
384
385
	mov	x0, sp
	mov	x1, #BAD_SYNC
	mrs	x2, esr_el1
	b	bad_mode
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
386
	enable_dbg
387
	TRACE_IRQSOFF
388

Catalin Marinas's avatar
Catalin Marinas committed
389
	irq_handler
390

391
#ifdef CONFIG_IPIPE
392
	bne	ipipe_fast_svc_irq_exit
393
#endif
Catalin Marinas's avatar
Catalin Marinas committed
394
#ifdef CONFIG_PREEMPT
395
	get_thread_info tsk
Neil Zhang's avatar
Neil Zhang committed
396
	ldr	w24, [tsk, #TI_PREEMPT]		// get preempt count
397
	cbnz	w24, 1f				// preempt count != 0
Catalin Marinas's avatar
Catalin Marinas committed
398
399
400
401
402
	ldr	x0, [tsk, #TI_FLAGS]		// get flags
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
403
#ifdef CONFIG_IPIPE
404
ipipe_fast_svc_irq_exit:
405
#endif
406
	TRACE_IRQSON
Catalin Marinas's avatar
Catalin Marinas committed
407
408
409
410
411
412
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
413
1:	bl	PREEMPT_SCHEDULE_IRQ
Catalin Marinas's avatar
Catalin Marinas committed
414
415
416
417
418
419
420
421
422
423
424
425
	ldr	x0, [tsk, #TI_FLAGS]		// get new tasks TI_FLAGS
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
426
427
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
Catalin Marinas's avatar
Catalin Marinas committed
428
	b.eq	el0_svc
Mark Rutland's avatar
Mark Rutland committed
429
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
430
	b.eq	el0_da
Mark Rutland's avatar
Mark Rutland committed
431
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
432
	b.eq	el0_ia
Mark Rutland's avatar
Mark Rutland committed
433
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
434
	b.eq	el0_fpsimd_acc
Mark Rutland's avatar
Mark Rutland committed
435
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
436
	b.eq	el0_fpsimd_exc
Mark Rutland's avatar
Mark Rutland committed
437
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
Catalin Marinas's avatar
Catalin Marinas committed
438
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
439
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
440
	b.eq	el0_sp_pc
Mark Rutland's avatar
Mark Rutland committed
441
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
442
	b.eq	el0_sp_pc
Mark Rutland's avatar
Mark Rutland committed
443
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
444
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
445
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
446
447
448
449
450
451
452
453
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
Mark Rutland's avatar
Mark Rutland committed
454
455
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
Catalin Marinas's avatar
Catalin Marinas committed
456
	b.eq	el0_svc_compat
Mark Rutland's avatar
Mark Rutland committed
457
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
458
	b.eq	el0_da
Mark Rutland's avatar
Mark Rutland committed
459
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
460
	b.eq	el0_ia
Mark Rutland's avatar
Mark Rutland committed
461
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
462
	b.eq	el0_fpsimd_acc
Mark Rutland's avatar
Mark Rutland committed
463
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
464
	b.eq	el0_fpsimd_exc
Mark Rutland's avatar
Mark Rutland committed
465
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
466
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
467
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
468
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
469
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
470
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
471
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
472
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
473
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
474
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
475
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
476
	b.eq	el0_undef
Mark Rutland's avatar
Mark Rutland committed
477
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
478
479
480
481
482
483
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
484
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
Catalin Marinas's avatar
Catalin Marinas committed
485
486
487
488
489
490
491
492
493
494
495
496
497
498
	uxtw	scno, w7			// syscall number in w7 (r7)
	mov     sc_nr, #__NR_compat_syscalls
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
499
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
500
	// enable interrupts before calling the main handler
501
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
502
	ct_user_exit
503
	bic	x0, x26, #(0xff << 56)
Catalin Marinas's avatar
Catalin Marinas committed
504
505
	mov	x1, x25
	mov	x2, sp
506
	bl	do_mem_abort
507
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
508
509
510
511
el0_ia:
	/*
	 * Instruction abort handling
	 */
512
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
513
	// enable interrupts before calling the main handler
514
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
515
	ct_user_exit
516
	mov	x0, x26
Catalin Marinas's avatar
Catalin Marinas committed
517
518
	orr	x1, x25, #1 << 24		// use reserved ISS bit for instruction aborts
	mov	x2, sp
519
	bl	do_mem_abort
520
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
521
522
523
524
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
525
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
526
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
527
528
	mov	x0, x25
	mov	x1, sp
529
	bl	do_fpsimd_acc
530
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
531
532
533
534
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
535
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
536
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
537
538
	mov	x0, x25
	mov	x1, sp
539
	bl	do_fpsimd_exc
540
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
541
542
543
544
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
545
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
546
	// enable interrupts before calling the main handler
547
	enable_dbg_and_irq
548
	ct_user_exit
549
	mov	x0, x26
Catalin Marinas's avatar
Catalin Marinas committed
550
551
	mov	x1, x25
	mov	x2, sp
552
	bl	do_sp_pc_abort
553
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
554
555
556
557
el0_undef:
	/*
	 * Undefined instruction
	 */
558
#ifdef CONFIG_IPIPE
559
560
561
	mov	x0, #7				// x0 = IPIPE_TRAP_UNDEFINSTR
	mov	x1, sp				// x1 = &regs
	bl	__ipipe_notify_trap		// branch to trap handler
562
	cmp	w0, #0
563
	bne	ret_from_exception
564
#endif /* CONFIG_IPIPE */
565
	// enable interrupts before calling the main handler
566
	enable_dbg_and_irq
Larry Bassel's avatar
Larry Bassel committed
567
	ct_user_exit
568
	mov	x0, sp
569
	bl	do_undefinstr
570
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
571
572
573
574
575
576
577
578
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
579
580
	bl	do_debug_exception
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
581
	ct_user_exit
582
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
583
el0_inv:
584
	enable_dbg
Larry Bassel's avatar
Larry Bassel committed
585
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
586
587
588
	mov	x0, sp
	mov	x1, #BAD_SYNC
	mrs	x2, esr_el1
589
	bl	bad_mode
590
	b	ret_from_exception
Catalin Marinas's avatar
Catalin Marinas committed
591
592
593
594
595
596
597
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
598
	TRACE_IRQSOFF
599

Larry Bassel's avatar
Larry Bassel committed
600
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
601
	irq_handler
602
#ifdef CONFIG_IPIPE
603
604
605
606
	b.eq	normal_irq_ret
	/* Fast IRQ exit, root domain stalled or not current. */
	kernel_exit 0, ret = 0
normal_irq_ret:
607
#endif	/* CONFIG_IPIPE */
608
	TRACE_IRQSON
Catalin Marinas's avatar
Catalin Marinas committed
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
	add	x8, x0, #THREAD_CPU_CONTEXT
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
	add	x8, x1, #THREAD_CPU_CONTEXT
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
	ret
ENDPROC(cpu_switch_to)

642
643
#ifdef CONFIG_IPIPE
ret_from_exception:
644
	disable_irq
645
646
	ldr	x0, [tsk, #TI_IPIPE]
	tst	x0, #_TIP_HEAD
647
	b.eq	ret_to_user_noirq
648
649
	kernel_exit 0, ret = 0
#endif /* CONFIG_IPIPE */
Catalin Marinas's avatar
Catalin Marinas committed
650
651
652
653
654
655
656
657
658
/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
	disable_irq				// disable interrupts
	ldr	x1, [tsk, #TI_FLAGS]
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, fast_work_pending
659
	enable_step_tsk x1, x2
Catalin Marinas's avatar
Catalin Marinas committed
660
661
662
663
664
665
666
667
668
	kernel_exit 0, ret = 1

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
fast_work_pending:
	str	x0, [sp, #S_X0]			// returned x0
work_pending:
	tbnz	x1, #TIF_NEED_RESCHED, work_resched
669
	/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
Catalin Marinas's avatar
Catalin Marinas committed
670
671
672
673
	ldr	x2, [sp, #S_PSTATE]
	mov	x0, sp				// 'regs'
	tst	x2, #PSR_MODE_MASK		// user mode regs?
	b.ne	no_work_pending			// returning to kernel
674
	enable_irq				// enable interrupts for do_notify_resume()
Catalin Marinas's avatar
Catalin Marinas committed
675
676
677
	bl	do_notify_resume
	b	ret_to_user
work_resched:
678
	enable_irq_cond
Catalin Marinas's avatar
Catalin Marinas committed
679
680
681
682
683
	bl	schedule

/*
 * "slow" syscall return path.
 */
684
ret_to_user:
Catalin Marinas's avatar
Catalin Marinas committed
685
	disable_irq				// disable interrupts
686
ret_to_user_noirq:
Catalin Marinas's avatar
Catalin Marinas committed
687
688
689
	ldr	x1, [tsk, #TI_FLAGS]
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
690
	enable_step_tsk x1, x2
Catalin Marinas's avatar
Catalin Marinas committed
691
692
693
694
695
696
697
698
no_work_pending:
	kernel_exit 0, ret = 0
ENDPROC(ret_to_user)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
699
	enable_irq_cond
Catalin Marinas's avatar
Catalin Marinas committed
700
	bl	schedule_tail
701
702
703
704
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
Catalin Marinas's avatar
Catalin Marinas committed
705
706
707
708
709
710
711
712
713
714
715
716
717
	b	ret_to_user
ENDPROC(ret_from_fork)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
	uxtw	scno, w8			// syscall number in w8
	mov	sc_nr, #__NR_syscalls
el0_svc_naked:					// compat entry point
	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
718
	enable_dbg_and_irq
719
720

#ifdef CONFIG_IPIPE
721
722
723
724
725
	ldr	x16, [tsk, #TI_IPIPE]
	tst     scno, __ARM_ipipe_syscall
	b.eq	fastcall_bypass
	tst	x16, #_TIP_HEAD
	b.eq	fastcall_bypass
726
	mov	x0, sp
727
	bl	ipipe_fastcall_hook
728
	cmp	w0, #0
729
730
731
732
733
734
	b.lt	no_fastcall
	ldr	x16, [tsk, #TI_IPIPE]
	tst	x16, #_TIP_HEAD
	b.ne	fastcall_exit
	bl	__ipipe_root_sync
fastcall_tail:
735
	ldr	x0, [sp, #S_X0]
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
	b	ret_fast_syscall
fastcall_exit:
	tst	x16, #_TIP_MAYDAY
	b.eq	fastcall_notail
	mov	x0, sp
	bl	__ipipe_call_mayday
fastcall_notail:
	ldr	x0, [sp, #S_X0]
	disable_irq
	ldr	x1, [tsk, #TI_FLAGS]
	enable_step_tsk x1, x2
	kernel_exit 0, ret = 1
no_fastcall:
	ldr	x16, [tsk, #TI_IPIPE]
fastcall_bypass:
	tst	x16, #_TIP_NOTIFY
	b.ne	syscall_pipeline
	tst     scno, __ARM_ipipe_syscall
	b.eq	regular_syscall
syscall_pipeline:
	mov	x0, sp
	bl	__ipipe_notify_syscall
	ldr	x16, [tsk, #TI_IPIPE]
	tst	x16, #_TIP_HEAD
	b.ne	fastcall_notail
	cmp	w0, #0
	b.ne	fastcall_tail
regular_syscall:
	ldp	x0, x1, [sp, #S_X0]
765
	ldp	x2, x3, [sp, #S_X2]
766
767
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
768
769
#endif /* CONFIG_IPIPE */

Larry Bassel's avatar
Larry Bassel committed
770
	ct_user_exit 1
Catalin Marinas's avatar
Catalin Marinas committed
771

772
773
774
	ldr	x16, [tsk, #TI_FLAGS]		// check for syscall hooks
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
775
776
777
	cmp     scno, sc_nr                     // check upper syscall limit
	b.hs	ni_sys
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
778
779
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
780
781
ni_sys:
	mov	x0, sp
782
783
	bl	do_ni_syscall
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
784
785
786
787
788
789
790
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
791
792
793
794
795
796
	mov	w0, #-1				// set default errno for
	cmp     scno, x0			// user-issued syscall(-1)
	b.ne	1f
	mov	x0, #-ENOSYS
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
797
	bl	syscall_trace_enter
798
799
	cmp	w0, #-1				// skip the syscall?
	b.eq	__sys_trace_return_skipped
Catalin Marinas's avatar
Catalin Marinas committed
800
801
802
	uxtw	scno, w0			// syscall number (possibly new)
	mov	x1, sp				// pointer to regs
	cmp	scno, sc_nr			// check upper syscall limit
803
	b.hs	__ni_sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
804
805
806
807
808
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
809
	blr	x16				// call sys_* routine
Catalin Marinas's avatar
Catalin Marinas committed
810
811

__sys_trace_return:
812
813
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
814
815
	mov	x0, sp
	bl	syscall_trace_exit
Catalin Marinas's avatar
Catalin Marinas committed
816
817
	b	ret_to_user

818
819
820
821
822
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

Catalin Marinas's avatar
Catalin Marinas committed
823
824
825
826
827
828
829
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)
830
831
832
833
834
835
836
837
838
839

#ifdef CONFIG_IPIPE_TRACE_MCOUNT
	.text
	.align 3
	.type mcount %function
	.global mcount
mcount:
	ret		// just return
ENDPROC(mcount)
#endif