entry-common.S 11.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
/*
 *  linux/arch/arm/kernel/entry-common.S
 *
 *  Copyright (C) 2000 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

11
#include <asm/assembler.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <asm/unistd.h>
13
#include <asm/ftrace.h>
14
#include <asm/unwind.h>
15
#include <asm/memory.h>
16
17
18
19
20
#ifdef CONFIG_AEABI
#include <asm/unistd-oabi.h>
#endif

	.equ	NR_syscalls, __NR_syscalls
Linus Torvalds's avatar
Linus Torvalds committed
21

22
23
24
25
26
27
28
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
	.macro  arch_ret_to_user, tmp1, tmp2
	.endm
#endif

Linus Torvalds's avatar
Linus Torvalds committed
29
30
#include "entry-header.S"

31
saved_psr	.req	r8
Russell King's avatar
Russell King committed
32
33
34
35
#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
saved_pc	.req	r9
#define TRACE(x...) x
#else
36
saved_pc	.req	lr
Russell King's avatar
Russell King committed
37
38
#define TRACE(x...)
#endif
Linus Torvalds's avatar
Linus Torvalds committed
39

40
	.section .entry.text,"ax",%progbits
Linus Torvalds's avatar
Linus Torvalds committed
41
	.align	5
Russell King's avatar
Russell King committed
42
#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING))
Linus Torvalds's avatar
Linus Torvalds committed
43
/*
Russell King's avatar
Russell King committed
44
45
46
47
 * This is the fast syscall return path.  We do as little as possible here,
 * such as avoiding writing r0 to the stack.  We only use this path if we
 * have tracing and context tracking disabled - the overheads from those
 * features make this path too inefficient.
Linus Torvalds's avatar
Linus Torvalds committed
48
49
 */
ret_fast_syscall:
50
51
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
Russell King's avatar
Russell King committed
52
	disable_irq_notrace			@ disable interrupts
53
54
55
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
56
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
57
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Linus Torvalds's avatar
Linus Torvalds committed
58
	bne	fast_work_pending
59

60

61
62
63
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr

64
	restore_user_regs fast = 1, offset = S_OFF
65
 UNWIND(.fnend		)
Russell King's avatar
Russell King committed
66
ENDPROC(ret_fast_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
67

Russell King's avatar
Russell King committed
68
	/* Ok, we need to do extra processing, enter the slow path. */
Linus Torvalds's avatar
Linus Torvalds committed
69
70
fast_work_pending:
	str	r0, [sp, #S_R0+S_OFF]!		@ returned r0
Russell King's avatar
Russell King committed
71
72
73
74
75
76
77
78
79
80
81
82
	/* fall through to work_pending */
#else
/*
 * The "replacement" ret_fast_syscall for when tracing or context tracking
 * is enabled.  As we will need to call out to some C functions, we save
 * r0 first to avoid needing to save registers around each C function call.
 */
ret_fast_syscall:
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
	disable_irq_notrace			@ disable interrupts
83
84
85
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Russell King's avatar
Russell King committed
86
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
87
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Russell King's avatar
Russell King committed
88
89
90
91
92
93
94
95
96
97
	beq	no_work_pending
 UNWIND(.fnend		)
ENDPROC(ret_fast_syscall)

	/* Slower path - fall through to work_pending */
#endif

	tst	r1, #_TIF_SYSCALL_WORK
	bne	__sys_trace_return_nosave
slow_work_pending:
Linus Torvalds's avatar
Linus Torvalds committed
98
99
	mov	r0, sp				@ 'regs'
	mov	r2, why				@ 'syscall'
100
	bl	do_work_pending
101
	cmp	r0, #0
102
	beq	no_work_pending
103
	movlt	scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
104
105
	ldmia	sp, {r0 - r6}			@ have to reload r0 - r6
	b	local_restart			@ ... and off we go
106
ENDPROC(ret_fast_syscall)
107

Linus Torvalds's avatar
Linus Torvalds committed
108
109
/*
 * "slow" syscall return path.  "why" tells us if this was a real syscall.
Russell King's avatar
Russell King committed
110
111
112
 * IRQs may be enabled here, so always disable them.  Note that we use the
 * "notrace" version to avoid calling into the tracing code unnecessarily.
 * do_work_pending() will update this state if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
113
114
115
 */
ENTRY(ret_to_user)
ret_slow_syscall:
Russell King's avatar
Russell King committed
116
	disable_irq_notrace			@ disable interrupts
117
ENTRY(ret_to_user_from_irq)
118
119
120
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Linus Torvalds's avatar
Linus Torvalds committed
121
122
	ldr	r1, [tsk, #TI_FLAGS]
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
123
	bne	slow_work_pending
Linus Torvalds's avatar
Linus Torvalds committed
124
no_work_pending:
Russell King's avatar
Russell King committed
125
	asm_trace_hardirqs_on save = 0
126

127
128
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr
129
	ct_user_enter save = 0
130

131
	restore_user_regs fast = 0, offset = 0
132
ENDPROC(ret_to_user_from_irq)
133
ENDPROC(ret_to_user)
Linus Torvalds's avatar
Linus Torvalds committed
134
135
136
137
138
139

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
140
141
	cmp	r5, #0
	movne	r0, r4
142
	badrne	lr, 1f
143
	retne	r5
144
1:	get_thread_info tsk
Linus Torvalds's avatar
Linus Torvalds committed
145
	b	ret_slow_syscall
146
ENDPROC(ret_from_fork)
Linus Torvalds's avatar
Linus Torvalds committed
147
148
149
150
151
152
153
154

/*=============================================================================
 * SWI handler
 *-----------------------------------------------------------------------------
 */

	.align	5
ENTRY(vector_swi)
155
156
157
#ifdef CONFIG_CPU_V7M
	v7m_exception_entry
#else
158
	sub	sp, sp, #PT_REGS_SIZE
159
	stmia	sp, {r0 - r12}			@ Calling r0 - r12
160
161
162
163
 ARM(	add	r8, sp, #S_PC		)
 ARM(	stmdb	r8, {sp, lr}^		)	@ Calling sp, lr
 THUMB(	mov	r8, sp			)
 THUMB(	store_user_sp_lr r8, r10, S_SP	)	@ calling sp, lr
164
	mrs	saved_psr, spsr			@ called from non-FIQ mode, so ok.
Russell King's avatar
Russell King committed
165
 TRACE(	mov	saved_pc, lr		)
166
167
	str	saved_pc, [sp, #S_PC]		@ Save calling PC
	str	saved_psr, [sp, #S_PSR]		@ Save CPSR
168
	str	r0, [sp, #S_OLD_R0]		@ Save OLD_R0
169
#endif
Linus Torvalds's avatar
Linus Torvalds committed
170
	zero_fp
171
	alignment_trap r10, ip, __cr_alignment
172
173
174
	asm_trace_hardirqs_on save=0
	enable_irq_notrace
	ct_user_exit save=0
175

176
177
178
	/*
	 * Get the system call number.
	 */
179

180
#if defined(CONFIG_OABI_COMPAT)
181

182
183
184
185
186
	/*
	 * If we have CONFIG_OABI_COMPAT then we need to look at the swi
	 * value to determine if it is an EABI or an old ABI call.
	 */
#ifdef CONFIG_ARM_THUMB
187
	tst	saved_psr, #PSR_T_BIT
188
	movne	r10, #0				@ no thumb OABI emulation
189
 USER(	ldreq	r10, [saved_pc, #-4]	)	@ get SWI instruction
190
#else
191
 USER(	ldr	r10, [saved_pc, #-4]	)	@ get SWI instruction
192
#endif
193
 ARM_BE8(rev	r10, r10)			@ little endian instruction
194
195
196
197
198
199

#elif defined(CONFIG_AEABI)

	/*
	 * Pure EABI user space always put syscall number into scno (r7).
	 */
200
#elif defined(CONFIG_ARM_THUMB)
201
	/* Legacy ABI only, possibly thumb mode. */
202
	tst	saved_psr, #PSR_T_BIT		@ this is SPSR from save_user_regs
203
	addne	scno, r7, #__NR_SYSCALL_BASE	@ put OS number in
204
 USER(	ldreq	scno, [saved_pc, #-4]	)
205

206
#else
207
	/* Legacy ABI only. */
208
 USER(	ldr	scno, [saved_pc, #-4]	)	@ get SWI instruction
209
#endif
Linus Torvalds's avatar
Linus Torvalds committed
210

211
212
	/* saved_psr and saved_pc are now dead */

213
214
	uaccess_disable tbl

215
216
217
218
219
220
221
222
223
224
225
226
227
	adr	tbl, sys_call_table		@ load syscall table pointer

#if defined(CONFIG_OABI_COMPAT)
	/*
	 * If the swi argument is zero, this is an EABI call and we do nothing.
	 *
	 * If this is an old ABI call, get the syscall number into scno and
	 * get the old ABI syscall table address.
	 */
	bics	r10, r10, #0xff000000
	eorne	scno, r10, #__NR_OABI_SYSCALL_BASE
	ldrne	tbl, =sys_oabi_call_table
#elif !defined(CONFIG_AEABI)
Linus Torvalds's avatar
Linus Torvalds committed
228
	bic	scno, scno, #0xff000000		@ mask off SWI op-code
229
	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
230
#endif
231
	get_thread_info tsk
232
233
234
235
236
	/*
	 * Reload the registers that may have been corrupted on entry to
	 * the syscall assembly (by tracing or context tracking.)
	 */
 TRACE(	ldmia	sp, {r0 - r3}		)
237

238
local_restart:
Nicolas Pitre's avatar
Nicolas Pitre committed
239
	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
240
	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
Nicolas Pitre's avatar
Nicolas Pitre committed
241

242
	tst	r10, #_TIF_SYSCALL_WORK		@ are we tracing syscalls?
Linus Torvalds's avatar
Linus Torvalds committed
243
244
245
	bne	__sys_trace

	cmp	scno, #NR_syscalls		@ check upper syscall limit
246
	badr	lr, ret_fast_syscall		@ return address
Linus Torvalds's avatar
Linus Torvalds committed
247
248
249
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine

	add	r1, sp, #S_OFF
250
2:	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
251
	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
252
	bcs	arm_syscall
253
	mov	why, #0				@ no longer a real syscall
Linus Torvalds's avatar
Linus Torvalds committed
254
	b	sys_ni_syscall			@ not private func
255
256
257
258
259
260
261
262
263
264
265

#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
	/*
	 * We failed to handle a fault trying to access the page
	 * containing the swi instruction, but we're not really in a
	 * position to return -EFAULT. Instead, return back to the
	 * instruction and re-enter the user fault handling path trying
	 * to page it in. This will likely result in sending SEGV to the
	 * current task.
	 */
9001:
266
	sub	lr, saved_pc, #4
267
	str	lr, [sp, #S_PC]
268
	get_thread_info tsk
269
270
	b	ret_fast_syscall
#endif
271
ENDPROC(vector_swi)
Linus Torvalds's avatar
Linus Torvalds committed
272
273
274
275
276
277

	/*
	 * This is the really slow path.  We're going to be doing
	 * context switches, and waiting for our parent to respond.
	 */
__sys_trace:
278
279
280
	mov	r1, scno
	add	r0, sp, #S_OFF
	bl	syscall_trace_enter
Linus Torvalds's avatar
Linus Torvalds committed
281

282
	badr	lr, __sys_trace_return		@ return address
283
	mov	scno, r0			@ syscall number (possibly new)
Linus Torvalds's avatar
Linus Torvalds committed
284
285
	add	r1, sp, #S_R0 + S_OFF		@ pointer to regs
	cmp	scno, #NR_syscalls		@ check upper syscall limit
286
287
	ldmccia	r1, {r0 - r6}			@ have to reload r0 - r6
	stmccia	sp, {r4, r5}			@ and update the stack args
Linus Torvalds's avatar
Linus Torvalds committed
288
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine
289
290
291
292
	cmp	scno, #-1			@ skip the syscall?
	bne	2b
	add	sp, sp, #S_OFF			@ restore stack
	b	ret_slow_syscall
Linus Torvalds's avatar
Linus Torvalds committed
293
294
295

__sys_trace_return:
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
296
297
	mov	r0, sp
	bl	syscall_trace_exit
Linus Torvalds's avatar
Linus Torvalds committed
298
299
	b	ret_slow_syscall

Russell King's avatar
Russell King committed
300
__sys_trace_return_nosave:
301
	enable_irq_notrace
Russell King's avatar
Russell King committed
302
303
304
305
	mov	r0, sp
	bl	syscall_trace_exit
	b	ret_slow_syscall

Linus Torvalds's avatar
Linus Torvalds committed
306
307
308
309
310
	.align	5
#ifdef CONFIG_ALIGNMENT_TRAP
	.type	__cr_alignment, #object
__cr_alignment:
	.word	cr_alignment
311
312
313
#endif
	.ltorg

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
	.macro	syscall_table_start, sym
	.equ	__sys_nr, 0
	.type	\sym, #object
ENTRY(\sym)
	.endm

	.macro	syscall, nr, func
	.ifgt	__sys_nr - \nr
	.error	"Duplicated/unorded system call entry"
	.endif
	.rept	\nr - __sys_nr
	.long	sys_ni_syscall
	.endr
	.long	\func
	.equ	__sys_nr, \nr + 1
	.endm

	.macro	syscall_table_end, sym
	.ifgt	__sys_nr - __NR_syscalls
	.error	"System call table too big"
	.endif
	.rept	__NR_syscalls - __sys_nr
	.long	sys_ni_syscall
	.endr
	.size	\sym, . - \sym
	.endm

#define NATIVE(nr, func) syscall nr, func

343
344
345
346
/*
 * This is the syscall table declaration for native ABI syscalls.
 * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 */
347
348
	syscall_table_start sys_call_table
#define COMPAT(nr, native, compat) syscall nr, native
349
#ifdef CONFIG_AEABI
350
#include <calls-eabi.S>
351
#else
352
#include <calls-oabi.S>
Linus Torvalds's avatar
Linus Torvalds committed
353
#endif
354
355
#undef COMPAT
	syscall_table_end sys_call_table
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
359
360

/*============================================================================
 * Special system call wrappers
 */
@ r0 = syscall number
361
@ r8 = syscall table
Linus Torvalds's avatar
Linus Torvalds committed
362
sys_syscall:
363
		bic	scno, r0, #__NR_OABI_SYSCALL_BASE
Linus Torvalds's avatar
Linus Torvalds committed
364
365
366
367
368
369
370
371
372
		cmp	scno, #__NR_syscall - __NR_SYSCALL_BASE
		cmpne	scno, #NR_syscalls	@ check range
		stmloia	sp, {r5, r6}		@ shuffle args
		movlo	r0, r1
		movlo	r1, r2
		movlo	r2, r3
		movlo	r3, r4
		ldrlo	pc, [tbl, scno, lsl #2]
		b	sys_ni_syscall
373
ENDPROC(sys_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
374
375
376

sys_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
377
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
378
		b	sys_sigreturn
379
ENDPROC(sys_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
380
381
382

sys_rt_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
383
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
384
		b	sys_rt_sigreturn
385
ENDPROC(sys_rt_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
386

387
388
389
390
sys_statfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_statfs64
391
ENDPROC(sys_statfs64_wrapper)
392
393
394
395
396

sys_fstatfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_fstatfs64
397
ENDPROC(sys_fstatfs64_wrapper)
398

Linus Torvalds's avatar
Linus Torvalds committed
399
400
401
402
403
404
/*
 * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 * offset, we return EINVAL.
 */
sys_mmap2:
		str	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
405
		b	sys_mmap_pgoff
406
ENDPROC(sys_mmap2)
407
408

#ifdef CONFIG_OABI_COMPAT
409

410
411
412
413
414
415
416
/*
 * These are syscalls with argument register differences
 */

sys_oabi_pread64:
		stmia	sp, {r3, r4}
		b	sys_pread64
417
ENDPROC(sys_oabi_pread64)
418
419
420
421

sys_oabi_pwrite64:
		stmia	sp, {r3, r4}
		b	sys_pwrite64
422
ENDPROC(sys_oabi_pwrite64)
423
424
425
426
427

sys_oabi_truncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_truncate64
428
ENDPROC(sys_oabi_truncate64)
429
430
431
432
433

sys_oabi_ftruncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_ftruncate64
434
ENDPROC(sys_oabi_ftruncate64)
435
436
437
438
439
440

sys_oabi_readahead:
		str	r3, [sp]
		mov	r3, r2
		mov	r2, r1
		b	sys_readahead
441
ENDPROC(sys_oabi_readahead)
442

443
444
445
446
/*
 * Let's declare a second syscall table for old ABI binaries
 * using the compatibility syscall entries.
 */
447
448
449
450
	syscall_table_start sys_oabi_call_table
#define COMPAT(nr, native, compat) syscall nr, compat
#include <calls-oabi.S>
	syscall_table_end sys_oabi_call_table
451

452
453
#endif