entry-common.S 12 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
/*
 *  linux/arch/arm/kernel/entry-common.S
 *
 *  Copyright (C) 2000 Russell King
5
 *  Copyright (C) 2005 Stelian Pop.
Linus Torvalds's avatar
Linus Torvalds committed
6
7
8
9
10
11
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

12
#include <asm/assembler.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <asm/unistd.h>
14
#include <asm/ftrace.h>
15
#include <asm/unwind.h>
16
#include <asm/memory.h>
17
18
19
20
21
#ifdef CONFIG_AEABI
#include <asm/unistd-oabi.h>
#endif

	.equ	NR_syscalls, __NR_syscalls
Linus Torvalds's avatar
Linus Torvalds committed
22

23
24
25
26
27
28
29
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
	.macro  arch_ret_to_user, tmp1, tmp2
	.endm
#endif

Linus Torvalds's avatar
Linus Torvalds committed
30
31
#include "entry-header.S"

32
saved_psr	.req	r8
Russell King's avatar
Russell King committed
33
34
35
36
#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
saved_pc	.req	r9
#define TRACE(x...) x
#else
37
saved_pc	.req	lr
Russell King's avatar
Russell King committed
38
39
#define TRACE(x...)
#endif
Linus Torvalds's avatar
Linus Torvalds committed
40

41
	.section .entry.text,"ax",%progbits
Linus Torvalds's avatar
Linus Torvalds committed
42
	.align	5
43
44
#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING) || \
	IS_ENABLED(CONFIG_DEBUG_RSEQ))
Linus Torvalds's avatar
Linus Torvalds committed
45
/*
Russell King's avatar
Russell King committed
46
47
 * This is the fast syscall return path.  We do as little as possible here,
 * such as avoiding writing r0 to the stack.  We only use this path if we
48
49
 * have tracing, context tracking and rseq debug disabled - the overheads
 * from those features make this path too inefficient.
Linus Torvalds's avatar
Linus Torvalds committed
50
51
 */
ret_fast_syscall:
52
__ret_fast_syscall:
53
54
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
Russell King's avatar
Russell King committed
55
	disable_irq_notrace			@ disable interrupts
56
57
58
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
59
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
60
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Linus Torvalds's avatar
Linus Torvalds committed
61
	bne	fast_work_pending
62

63
64
65
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr

66
	restore_user_regs fast = 1, offset = S_OFF
67
68


69
 UNWIND(.fnend		)
Russell King's avatar
Russell King committed
70
ENDPROC(ret_fast_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
71

Russell King's avatar
Russell King committed
72
	/* Ok, we need to do extra processing, enter the slow path. */
Linus Torvalds's avatar
Linus Torvalds committed
73
74
fast_work_pending:
	str	r0, [sp, #S_R0+S_OFF]!		@ returned r0
Russell King's avatar
Russell King committed
75
76
77
	/* fall through to work_pending */
#else
/*
78
79
80
81
 * The "replacement" ret_fast_syscall for when tracing, context tracking,
 * or rseq debug is enabled.  As we will need to call out to some C functions,
 * we save r0 first to avoid needing to save registers around each C function
 * call.
Russell King's avatar
Russell King committed
82
83
 */
ret_fast_syscall:
84
__ret_fast_syscall:
Russell King's avatar
Russell King committed
85
86
87
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
88
89
90
91
92
#if IS_ENABLED(CONFIG_DEBUG_RSEQ)
	/* do_rseq_syscall needs interrupts enabled. */
	mov	r0, sp				@ 'regs'
	bl	do_rseq_syscall
#endif
Russell King's avatar
Russell King committed
93
	disable_irq_notrace			@ disable interrupts
94
95
96
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Russell King's avatar
Russell King committed
97
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
98
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Russell King's avatar
Russell King committed
99
100
101
102
103
104
105
106
107
108
	beq	no_work_pending
 UNWIND(.fnend		)
ENDPROC(ret_fast_syscall)

	/* Slower path - fall through to work_pending */
#endif

	tst	r1, #_TIF_SYSCALL_WORK
	bne	__sys_trace_return_nosave
slow_work_pending:
Linus Torvalds's avatar
Linus Torvalds committed
109
110
	mov	r0, sp				@ 'regs'
	mov	r2, why				@ 'syscall'
111
	bl	do_work_pending
112
	cmp	r0, #0
113
	beq	no_work_pending
114
	movlt	scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
115
116
	ldmia	sp, {r0 - r6}			@ have to reload r0 - r6
	b	local_restart			@ ... and off we go
117
ENDPROC(ret_fast_syscall)
118

Linus Torvalds's avatar
Linus Torvalds committed
119
120
/*
 * "slow" syscall return path.  "why" tells us if this was a real syscall.
Russell King's avatar
Russell King committed
121
122
123
 * IRQs may be enabled here, so always disable them.  Note that we use the
 * "notrace" version to avoid calling into the tracing code unnecessarily.
 * do_work_pending() will update this state if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
124
125
126
 */
ENTRY(ret_to_user)
ret_slow_syscall:
127
128
129
130
131
132
#if IS_ENABLED(CONFIG_DEBUG_RSEQ)
	/* do_rseq_syscall needs interrupts enabled. */
	enable_irq_notrace			@ enable interrupts
	mov	r0, sp				@ 'regs'
	bl	do_rseq_syscall
#endif
Russell King's avatar
Russell King committed
133
	disable_irq_notrace			@ disable interrupts
134
ENTRY(ret_to_user_from_irq)
135
136
137
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Linus Torvalds's avatar
Linus Torvalds committed
138
139
	ldr	r1, [tsk, #TI_FLAGS]
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
140
	bne	slow_work_pending
Linus Torvalds's avatar
Linus Torvalds committed
141
no_work_pending:
Russell King's avatar
Russell King committed
142
	asm_trace_hardirqs_on save = 0
143
	slow_restore_user_regs
144
ENDPROC(ret_to_user_from_irq)
145
ENDPROC(ret_to_user)
Linus Torvalds's avatar
Linus Torvalds committed
146
147
148
149
150

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
151
	enable_irq_cond
Linus Torvalds's avatar
Linus Torvalds committed
152
	bl	schedule_tail
153
154
	cmp	r5, #0
	movne	r0, r4
155
	badrne	lr, 1f
156
	retne	r5
157
1:	get_thread_info tsk
Linus Torvalds's avatar
Linus Torvalds committed
158
	b	ret_slow_syscall
159
ENDPROC(ret_from_fork)
Linus Torvalds's avatar
Linus Torvalds committed
160

161
162
163
164
165
166
#ifdef CONFIG_IPIPE
ENTRY(__ipipe_ret_to_user_irqs_disabled)
	slow_restore_user_regs
ENDPROC(__ipipe_ret_to_user_irqs_disabled)
#endif

Linus Torvalds's avatar
Linus Torvalds committed
167
168
169
170
171
172
173
/*=============================================================================
 * SWI handler
 *-----------------------------------------------------------------------------
 */

	.align	5
ENTRY(vector_swi)
174
175
176
#ifdef CONFIG_CPU_V7M
	v7m_exception_entry
#else
177
	sub	sp, sp, #PT_REGS_SIZE
178
	stmia	sp, {r0 - r12}			@ Calling r0 - r12
179
180
181
182
 ARM(	add	r8, sp, #S_PC		)
 ARM(	stmdb	r8, {sp, lr}^		)	@ Calling sp, lr
 THUMB(	mov	r8, sp			)
 THUMB(	store_user_sp_lr r8, r10, S_SP	)	@ calling sp, lr
183
	mrs	saved_psr, spsr			@ called from non-FIQ mode, so ok.
Russell King's avatar
Russell King committed
184
 TRACE(	mov	saved_pc, lr		)
185
186
	str	saved_pc, [sp, #S_PC]		@ Save calling PC
	str	saved_psr, [sp, #S_PSR]		@ Save CPSR
187
	str	r0, [sp, #S_OLD_R0]		@ Save OLD_R0
188
#endif
Linus Torvalds's avatar
Linus Torvalds committed
189
	zero_fp
190
191
192
193
194
195
196
197
198
199
#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
	mov	r4, lr
	mov	r0, #1		/* IPIPE_TRACE_BEGIN */
	mov	r3, #0x90000000
	sub	r2, lr, #4	/* calling PC */
	mov	r1, pc
	bl	ipipe_trace_asm
	mov	lr, r4
	ldm	sp, {r0 - r4}
#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
200
	alignment_trap r10, ip, __cr_alignment
201
202
203
	asm_trace_hardirqs_on save=0
	enable_irq_notrace
	ct_user_exit save=0
204

205
206
207
	/*
	 * Get the system call number.
	 */
208

209
#if defined(CONFIG_OABI_COMPAT)
210

211
212
213
214
215
	/*
	 * If we have CONFIG_OABI_COMPAT then we need to look at the swi
	 * value to determine if it is an EABI or an old ABI call.
	 */
#ifdef CONFIG_ARM_THUMB
216
	tst	saved_psr, #PSR_T_BIT
217
	movne	r10, #0				@ no thumb OABI emulation
218
 USER(	ldreq	r10, [saved_pc, #-4]	)	@ get SWI instruction
219
#else
220
 USER(	ldr	r10, [saved_pc, #-4]	)	@ get SWI instruction
221
#endif
222
 ARM_BE8(rev	r10, r10)			@ little endian instruction
223
224
225
226
227
228

#elif defined(CONFIG_AEABI)

	/*
	 * Pure EABI user space always put syscall number into scno (r7).
	 */
229
#elif defined(CONFIG_ARM_THUMB)
230
	/* Legacy ABI only, possibly thumb mode. */
231
	tst	saved_psr, #PSR_T_BIT		@ this is SPSR from save_user_regs
232
	addne	scno, r7, #__NR_SYSCALL_BASE	@ put OS number in
233
 USER(	ldreq	scno, [saved_pc, #-4]	)
234

235
#else
236
	/* Legacy ABI only. */
237
 USER(	ldr	scno, [saved_pc, #-4]	)	@ get SWI instruction
238
#endif
Linus Torvalds's avatar
Linus Torvalds committed
239

240
241
	/* saved_psr and saved_pc are now dead */

242
243
	uaccess_disable tbl

244
245
246
247
248
249
250
251
252
253
254
255
256
	adr	tbl, sys_call_table		@ load syscall table pointer

#if defined(CONFIG_OABI_COMPAT)
	/*
	 * If the swi argument is zero, this is an EABI call and we do nothing.
	 *
	 * If this is an old ABI call, get the syscall number into scno and
	 * get the old ABI syscall table address.
	 */
	bics	r10, r10, #0xff000000
	eorne	scno, r10, #__NR_OABI_SYSCALL_BASE
	ldrne	tbl, =sys_oabi_call_table
#elif !defined(CONFIG_AEABI)
Linus Torvalds's avatar
Linus Torvalds committed
257
	bic	scno, scno, #0xff000000		@ mask off SWI op-code
258
	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
259
#endif
260
	get_thread_info tsk
261
262
263
264
265
	/*
	 * Reload the registers that may have been corrupted on entry to
	 * the syscall assembly (by tracing or context tracking.)
	 */
 TRACE(	ldmia	sp, {r0 - r3}		)
266

267
local_restart:
Nicolas Pitre's avatar
Nicolas Pitre committed
268
	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
269
	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
Nicolas Pitre's avatar
Nicolas Pitre committed
270

271
	tst	r10, #_TIF_SYSCALL_WORK		@ are we tracing syscalls?
Linus Torvalds's avatar
Linus Torvalds committed
272
273
	bne	__sys_trace

274
	invoke_syscall tbl, scno, r10, __ret_fast_syscall
Linus Torvalds's avatar
Linus Torvalds committed
275
276

	add	r1, sp, #S_OFF
277
2:	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
278
	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
279
	bcs	arm_syscall
280
	mov	why, #0				@ no longer a real syscall
Linus Torvalds's avatar
Linus Torvalds committed
281
	b	sys_ni_syscall			@ not private func
282
283
284
285
286
287
288
289
290
291
292

#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
	/*
	 * We failed to handle a fault trying to access the page
	 * containing the swi instruction, but we're not really in a
	 * position to return -EFAULT. Instead, return back to the
	 * instruction and re-enter the user fault handling path trying
	 * to page it in. This will likely result in sending SEGV to the
	 * current task.
	 */
9001:
293
	sub	lr, saved_pc, #4
294
	str	lr, [sp, #S_PC]
295
	get_thread_info tsk
296
297
	b	ret_fast_syscall
#endif
298
ENDPROC(vector_swi)
Linus Torvalds's avatar
Linus Torvalds committed
299
300
301
302
303
304

	/*
	 * This is the really slow path.  We're going to be doing
	 * context switches, and waiting for our parent to respond.
	 */
__sys_trace:
305
306
307
	mov	r1, scno
	add	r0, sp, #S_OFF
	bl	syscall_trace_enter
308
309
	mov	scno, r0
	invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1
310
311
312
313
	cmp	scno, #-1			@ skip the syscall?
	bne	2b
	add	sp, sp, #S_OFF			@ restore stack
	b	ret_slow_syscall
Linus Torvalds's avatar
Linus Torvalds committed
314
315
316

__sys_trace_return:
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
317
318
	mov	r0, sp
	bl	syscall_trace_exit
Linus Torvalds's avatar
Linus Torvalds committed
319
320
	b	ret_slow_syscall

Russell King's avatar
Russell King committed
321
__sys_trace_return_nosave:
322
	enable_irq_notrace
Russell King's avatar
Russell King committed
323
324
325
326
	mov	r0, sp
	bl	syscall_trace_exit
	b	ret_slow_syscall

Linus Torvalds's avatar
Linus Torvalds committed
327
328
329
330
331
	.align	5
#ifdef CONFIG_ALIGNMENT_TRAP
	.type	__cr_alignment, #object
__cr_alignment:
	.word	cr_alignment
332
333
334
#endif
	.ltorg

335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
	.macro	syscall_table_start, sym
	.equ	__sys_nr, 0
	.type	\sym, #object
ENTRY(\sym)
	.endm

	.macro	syscall, nr, func
	.ifgt	__sys_nr - \nr
	.error	"Duplicated/unorded system call entry"
	.endif
	.rept	\nr - __sys_nr
	.long	sys_ni_syscall
	.endr
	.long	\func
	.equ	__sys_nr, \nr + 1
	.endm

	.macro	syscall_table_end, sym
	.ifgt	__sys_nr - __NR_syscalls
	.error	"System call table too big"
	.endif
	.rept	__NR_syscalls - __sys_nr
	.long	sys_ni_syscall
	.endr
	.size	\sym, . - \sym
	.endm

#define NATIVE(nr, func) syscall nr, func

364
365
366
367
/*
 * This is the syscall table declaration for native ABI syscalls.
 * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 */
368
369
	syscall_table_start sys_call_table
#define COMPAT(nr, native, compat) syscall nr, native
370
#ifdef CONFIG_AEABI
371
#include <calls-eabi.S>
372
#else
373
#include <calls-oabi.S>
Linus Torvalds's avatar
Linus Torvalds committed
374
#endif
375
376
#undef COMPAT
	syscall_table_end sys_call_table
Linus Torvalds's avatar
Linus Torvalds committed
377
378
379
380
381

/*============================================================================
 * Special system call wrappers
 */
@ r0 = syscall number
382
@ r8 = syscall table
Linus Torvalds's avatar
Linus Torvalds committed
383
sys_syscall:
384
		bic	scno, r0, #__NR_OABI_SYSCALL_BASE
Linus Torvalds's avatar
Linus Torvalds committed
385
386
		cmp	scno, #__NR_syscall - __NR_SYSCALL_BASE
		cmpne	scno, #NR_syscalls	@ check range
387
388
389
390
#ifdef CONFIG_CPU_SPECTRE
		movhs	scno, #0
		csdb
#endif
Linus Torvalds's avatar
Linus Torvalds committed
391
392
393
394
395
396
397
		stmloia	sp, {r5, r6}		@ shuffle args
		movlo	r0, r1
		movlo	r1, r2
		movlo	r2, r3
		movlo	r3, r4
		ldrlo	pc, [tbl, scno, lsl #2]
		b	sys_ni_syscall
398
ENDPROC(sys_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
399
400
401

sys_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
402
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
403
		b	sys_sigreturn
404
ENDPROC(sys_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
405
406
407

sys_rt_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
408
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
409
		b	sys_rt_sigreturn
410
ENDPROC(sys_rt_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
411

412
413
414
415
sys_statfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_statfs64
416
ENDPROC(sys_statfs64_wrapper)
417
418
419
420
421

sys_fstatfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_fstatfs64
422
ENDPROC(sys_fstatfs64_wrapper)
423

Linus Torvalds's avatar
Linus Torvalds committed
424
425
426
427
428
429
/*
 * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 * offset, we return EINVAL.
 */
sys_mmap2:
		str	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
430
		b	sys_mmap_pgoff
431
ENDPROC(sys_mmap2)
432
433

#ifdef CONFIG_OABI_COMPAT
434

435
436
437
438
439
440
441
/*
 * These are syscalls with argument register differences
 */

sys_oabi_pread64:
		stmia	sp, {r3, r4}
		b	sys_pread64
442
ENDPROC(sys_oabi_pread64)
443
444
445
446

sys_oabi_pwrite64:
		stmia	sp, {r3, r4}
		b	sys_pwrite64
447
ENDPROC(sys_oabi_pwrite64)
448
449
450
451
452

sys_oabi_truncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_truncate64
453
ENDPROC(sys_oabi_truncate64)
454
455
456
457
458

sys_oabi_ftruncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_ftruncate64
459
ENDPROC(sys_oabi_ftruncate64)
460
461
462
463
464
465

sys_oabi_readahead:
		str	r3, [sp]
		mov	r3, r2
		mov	r2, r1
		b	sys_readahead
466
ENDPROC(sys_oabi_readahead)
467

468
469
470
471
/*
 * Let's declare a second syscall table for old ABI binaries
 * using the compatibility syscall entries.
 */
472
473
474
475
	syscall_table_start sys_oabi_call_table
#define COMPAT(nr, native, compat) syscall nr, compat
#include <calls-oabi.S>
	syscall_table_end sys_oabi_call_table
476

477
478
#endif

479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
#if defined(CONFIG_FRAME_POINTER) && (CONFIG_IPIPE_TRACE)

	.text
	.align 0
	.type arm_return_addr %function
	.global arm_return_addr

arm_return_addr:
	mov	ip, r0
	mov	r0, fp
3:
	cmp	r0, #0
	beq	1f		@ frame list hit end, bail
	cmp	ip, #0
	beq	2f		@ reached desired frame
	ldr	r0, [r0, #-12]  @ else continue, get next fp
	sub	ip, ip, #1
	b	3b
2:
	ldr	r0, [r0, #-4]   @ get target return address
1:
	mov	pc, lr

#endif