entry-common.S 11.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
/*
 *  linux/arch/arm/kernel/entry-common.S
 *
 *  Copyright (C) 2000 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

11
#include <asm/assembler.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <asm/unistd.h>
13
#include <asm/ftrace.h>
14
#include <asm/unwind.h>
15
16
17
18
19
#ifdef CONFIG_AEABI
#include <asm/unistd-oabi.h>
#endif

	.equ	NR_syscalls, __NR_syscalls
Linus Torvalds's avatar
Linus Torvalds committed
20

21
22
23
24
25
26
27
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
	.macro  arch_ret_to_user, tmp1, tmp2
	.endm
#endif

Linus Torvalds's avatar
Linus Torvalds committed
28
29
#include "entry-header.S"

30
saved_psr	.req	r8
Russell King's avatar
Russell King committed
31
32
33
34
#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
saved_pc	.req	r9
#define TRACE(x...) x
#else
35
saved_pc	.req	lr
Russell King's avatar
Russell King committed
36
37
#define TRACE(x...)
#endif
Linus Torvalds's avatar
Linus Torvalds committed
38
39

	.align	5
Russell King's avatar
Russell King committed
40
#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING))
Linus Torvalds's avatar
Linus Torvalds committed
41
/*
Russell King's avatar
Russell King committed
42
43
44
45
 * This is the fast syscall return path.  We do as little as possible here,
 * such as avoiding writing r0 to the stack.  We only use this path if we
 * have tracing and context tracking disabled - the overheads from those
 * features make this path too inefficient.
Linus Torvalds's avatar
Linus Torvalds committed
46
47
 */
ret_fast_syscall:
48
49
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
Russell King's avatar
Russell King committed
50
	disable_irq_notrace			@ disable interrupts
51
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
52
53
54
	tst	r1, #_TIF_SYSCALL_WORK
	bne	fast_work_pending
	tst	r1, #_TIF_WORK_MASK
Linus Torvalds's avatar
Linus Torvalds committed
55
	bne	fast_work_pending
56

57
58
59
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr

60
	restore_user_regs fast = 1, offset = S_OFF
61
 UNWIND(.fnend		)
Russell King's avatar
Russell King committed
62
ENDPROC(ret_fast_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
63

Russell King's avatar
Russell King committed
64
	/* Ok, we need to do extra processing, enter the slow path. */
Linus Torvalds's avatar
Linus Torvalds committed
65
66
fast_work_pending:
	str	r0, [sp, #S_R0+S_OFF]!		@ returned r0
Russell King's avatar
Russell King committed
67
68
69
70
71
72
73
74
75
76
77
78
79
	/* fall through to work_pending */
#else
/*
 * The "replacement" ret_fast_syscall for when tracing or context tracking
 * is enabled.  As we will need to call out to some C functions, we save
 * r0 first to avoid needing to save registers around each C function call.
 */
ret_fast_syscall:
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
	disable_irq_notrace			@ disable interrupts
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
80
81
82
	tst	r1, #_TIF_SYSCALL_WORK
	bne	fast_work_pending
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
83
84
85
86
87
	beq	no_work_pending
 UNWIND(.fnend		)
ENDPROC(ret_fast_syscall)

	/* Slower path - fall through to work_pending */
88
fast_work_pending:
Russell King's avatar
Russell King committed
89
90
91
92
93
#endif

	tst	r1, #_TIF_SYSCALL_WORK
	bne	__sys_trace_return_nosave
slow_work_pending:
Linus Torvalds's avatar
Linus Torvalds committed
94
95
	mov	r0, sp				@ 'regs'
	mov	r2, why				@ 'syscall'
96
	bl	do_work_pending
97
	cmp	r0, #0
98
	beq	no_work_pending
99
	movlt	scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
100
101
	ldmia	sp, {r0 - r6}			@ have to reload r0 - r6
	b	local_restart			@ ... and off we go
102
ENDPROC(ret_fast_syscall)
103

Linus Torvalds's avatar
Linus Torvalds committed
104
105
/*
 * "slow" syscall return path.  "why" tells us if this was a real syscall.
Russell King's avatar
Russell King committed
106
107
108
 * IRQs may be enabled here, so always disable them.  Note that we use the
 * "notrace" version to avoid calling into the tracing code unnecessarily.
 * do_work_pending() will update this state if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
109
110
111
 */
ENTRY(ret_to_user)
ret_slow_syscall:
Russell King's avatar
Russell King committed
112
	disable_irq_notrace			@ disable interrupts
113
ENTRY(ret_to_user_from_irq)
Linus Torvalds's avatar
Linus Torvalds committed
114
115
	ldr	r1, [tsk, #TI_FLAGS]
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
116
	bne	slow_work_pending
Linus Torvalds's avatar
Linus Torvalds committed
117
no_work_pending:
Russell King's avatar
Russell King committed
118
	asm_trace_hardirqs_on save = 0
119

120
121
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr
122
	ct_user_enter save = 0
123

124
	restore_user_regs fast = 0, offset = 0
125
ENDPROC(ret_to_user_from_irq)
126
ENDPROC(ret_to_user)
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
130
131
132

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
133
134
	cmp	r5, #0
	movne	r0, r4
135
	badrne	lr, 1f
136
	retne	r5
137
1:	get_thread_info tsk
Linus Torvalds's avatar
Linus Torvalds committed
138
	b	ret_slow_syscall
139
ENDPROC(ret_from_fork)
Linus Torvalds's avatar
Linus Torvalds committed
140
141
142
143
144
145
146
147

/*=============================================================================
 * SWI handler
 *-----------------------------------------------------------------------------
 */

	.align	5
ENTRY(vector_swi)
148
149
150
#ifdef CONFIG_CPU_V7M
	v7m_exception_entry
#else
151
	sub	sp, sp, #PT_REGS_SIZE
152
	stmia	sp, {r0 - r12}			@ Calling r0 - r12
153
154
155
156
 ARM(	add	r8, sp, #S_PC		)
 ARM(	stmdb	r8, {sp, lr}^		)	@ Calling sp, lr
 THUMB(	mov	r8, sp			)
 THUMB(	store_user_sp_lr r8, r10, S_SP	)	@ calling sp, lr
157
	mrs	saved_psr, spsr			@ called from non-FIQ mode, so ok.
Russell King's avatar
Russell King committed
158
 TRACE(	mov	saved_pc, lr		)
159
160
	str	saved_pc, [sp, #S_PC]		@ Save calling PC
	str	saved_psr, [sp, #S_PSR]		@ Save CPSR
161
	str	r0, [sp, #S_OLD_R0]		@ Save OLD_R0
162
#endif
Linus Torvalds's avatar
Linus Torvalds committed
163
	zero_fp
164
	alignment_trap r10, ip, __cr_alignment
165
166
167
	asm_trace_hardirqs_on save=0
	enable_irq_notrace
	ct_user_exit save=0
168

169
170
171
	/*
	 * Get the system call number.
	 */
172

173
#if defined(CONFIG_OABI_COMPAT)
174

175
176
177
178
179
	/*
	 * If we have CONFIG_OABI_COMPAT then we need to look at the swi
	 * value to determine if it is an EABI or an old ABI call.
	 */
#ifdef CONFIG_ARM_THUMB
180
	tst	saved_psr, #PSR_T_BIT
181
	movne	r10, #0				@ no thumb OABI emulation
182
 USER(	ldreq	r10, [saved_pc, #-4]	)	@ get SWI instruction
183
#else
184
 USER(	ldr	r10, [saved_pc, #-4]	)	@ get SWI instruction
185
#endif
186
 ARM_BE8(rev	r10, r10)			@ little endian instruction
187
188
189
190
191
192

#elif defined(CONFIG_AEABI)

	/*
	 * Pure EABI user space always put syscall number into scno (r7).
	 */
193
#elif defined(CONFIG_ARM_THUMB)
194
	/* Legacy ABI only, possibly thumb mode. */
195
	tst	saved_psr, #PSR_T_BIT		@ this is SPSR from save_user_regs
196
	addne	scno, r7, #__NR_SYSCALL_BASE	@ put OS number in
197
 USER(	ldreq	scno, [saved_pc, #-4]	)
198

199
#else
200
	/* Legacy ABI only. */
201
 USER(	ldr	scno, [saved_pc, #-4]	)	@ get SWI instruction
202
#endif
Linus Torvalds's avatar
Linus Torvalds committed
203

204
205
	/* saved_psr and saved_pc are now dead */

206
207
	uaccess_disable tbl

208
209
210
211
212
213
214
215
216
217
218
219
220
	adr	tbl, sys_call_table		@ load syscall table pointer

#if defined(CONFIG_OABI_COMPAT)
	/*
	 * If the swi argument is zero, this is an EABI call and we do nothing.
	 *
	 * If this is an old ABI call, get the syscall number into scno and
	 * get the old ABI syscall table address.
	 */
	bics	r10, r10, #0xff000000
	eorne	scno, r10, #__NR_OABI_SYSCALL_BASE
	ldrne	tbl, =sys_oabi_call_table
#elif !defined(CONFIG_AEABI)
Linus Torvalds's avatar
Linus Torvalds committed
221
	bic	scno, scno, #0xff000000		@ mask off SWI op-code
222
	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
223
#endif
224
	get_thread_info tsk
225
226
227
228
229
	/*
	 * Reload the registers that may have been corrupted on entry to
	 * the syscall assembly (by tracing or context tracking.)
	 */
 TRACE(	ldmia	sp, {r0 - r3}		)
230

231
local_restart:
Nicolas Pitre's avatar
Nicolas Pitre committed
232
	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
233
	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
Nicolas Pitre's avatar
Nicolas Pitre committed
234

235
	tst	r10, #_TIF_SYSCALL_WORK		@ are we tracing syscalls?
Linus Torvalds's avatar
Linus Torvalds committed
236
237
238
	bne	__sys_trace

	cmp	scno, #NR_syscalls		@ check upper syscall limit
239
	badr	lr, ret_fast_syscall		@ return address
Linus Torvalds's avatar
Linus Torvalds committed
240
241
242
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine

	add	r1, sp, #S_OFF
243
2:	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
244
	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
245
	bcs	arm_syscall
246
	mov	why, #0				@ no longer a real syscall
Linus Torvalds's avatar
Linus Torvalds committed
247
	b	sys_ni_syscall			@ not private func
248
249
250
251
252
253
254
255
256
257
258

#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
	/*
	 * We failed to handle a fault trying to access the page
	 * containing the swi instruction, but we're not really in a
	 * position to return -EFAULT. Instead, return back to the
	 * instruction and re-enter the user fault handling path trying
	 * to page it in. This will likely result in sending SEGV to the
	 * current task.
	 */
9001:
259
	sub	lr, saved_pc, #4
260
	str	lr, [sp, #S_PC]
261
	get_thread_info tsk
262
263
	b	ret_fast_syscall
#endif
264
ENDPROC(vector_swi)
Linus Torvalds's avatar
Linus Torvalds committed
265
266
267
268
269
270

	/*
	 * This is the really slow path.  We're going to be doing
	 * context switches, and waiting for our parent to respond.
	 */
__sys_trace:
271
272
273
	mov	r1, scno
	add	r0, sp, #S_OFF
	bl	syscall_trace_enter
Linus Torvalds's avatar
Linus Torvalds committed
274

275
	badr	lr, __sys_trace_return		@ return address
276
	mov	scno, r0			@ syscall number (possibly new)
Linus Torvalds's avatar
Linus Torvalds committed
277
278
	add	r1, sp, #S_R0 + S_OFF		@ pointer to regs
	cmp	scno, #NR_syscalls		@ check upper syscall limit
279
280
	ldmccia	r1, {r0 - r6}			@ have to reload r0 - r6
	stmccia	sp, {r4, r5}			@ and update the stack args
Linus Torvalds's avatar
Linus Torvalds committed
281
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine
282
283
284
285
	cmp	scno, #-1			@ skip the syscall?
	bne	2b
	add	sp, sp, #S_OFF			@ restore stack
	b	ret_slow_syscall
Linus Torvalds's avatar
Linus Torvalds committed
286
287
288

__sys_trace_return:
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
289
290
	mov	r0, sp
	bl	syscall_trace_exit
Linus Torvalds's avatar
Linus Torvalds committed
291
292
	b	ret_slow_syscall

Russell King's avatar
Russell King committed
293
__sys_trace_return_nosave:
294
	enable_irq_notrace
Russell King's avatar
Russell King committed
295
296
297
298
	mov	r0, sp
	bl	syscall_trace_exit
	b	ret_slow_syscall

Linus Torvalds's avatar
Linus Torvalds committed
299
300
301
302
303
	.align	5
#ifdef CONFIG_ALIGNMENT_TRAP
	.type	__cr_alignment, #object
__cr_alignment:
	.word	cr_alignment
304
305
306
#endif
	.ltorg

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
	.macro	syscall_table_start, sym
	.equ	__sys_nr, 0
	.type	\sym, #object
ENTRY(\sym)
	.endm

	.macro	syscall, nr, func
	.ifgt	__sys_nr - \nr
	.error	"Duplicated/unorded system call entry"
	.endif
	.rept	\nr - __sys_nr
	.long	sys_ni_syscall
	.endr
	.long	\func
	.equ	__sys_nr, \nr + 1
	.endm

	.macro	syscall_table_end, sym
	.ifgt	__sys_nr - __NR_syscalls
	.error	"System call table too big"
	.endif
	.rept	__NR_syscalls - __sys_nr
	.long	sys_ni_syscall
	.endr
	.size	\sym, . - \sym
	.endm

#define NATIVE(nr, func) syscall nr, func

336
337
338
339
/*
 * This is the syscall table declaration for native ABI syscalls.
 * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 */
340
341
	syscall_table_start sys_call_table
#define COMPAT(nr, native, compat) syscall nr, native
342
#ifdef CONFIG_AEABI
343
#include <calls-eabi.S>
344
#else
345
#include <calls-oabi.S>
Linus Torvalds's avatar
Linus Torvalds committed
346
#endif
347
348
#undef COMPAT
	syscall_table_end sys_call_table
Linus Torvalds's avatar
Linus Torvalds committed
349
350
351
352
353

/*============================================================================
 * Special system call wrappers
 */
@ r0 = syscall number
354
@ r8 = syscall table
Linus Torvalds's avatar
Linus Torvalds committed
355
sys_syscall:
356
		bic	scno, r0, #__NR_OABI_SYSCALL_BASE
Linus Torvalds's avatar
Linus Torvalds committed
357
358
359
360
361
362
363
364
365
		cmp	scno, #__NR_syscall - __NR_SYSCALL_BASE
		cmpne	scno, #NR_syscalls	@ check range
		stmloia	sp, {r5, r6}		@ shuffle args
		movlo	r0, r1
		movlo	r1, r2
		movlo	r2, r3
		movlo	r3, r4
		ldrlo	pc, [tbl, scno, lsl #2]
		b	sys_ni_syscall
366
ENDPROC(sys_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
367
368
369

sys_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
370
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
371
		b	sys_sigreturn
372
ENDPROC(sys_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
373
374
375

sys_rt_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
376
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
377
		b	sys_rt_sigreturn
378
ENDPROC(sys_rt_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
379

380
381
382
383
sys_statfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_statfs64
384
ENDPROC(sys_statfs64_wrapper)
385
386
387
388
389

sys_fstatfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_fstatfs64
390
ENDPROC(sys_fstatfs64_wrapper)
391

Linus Torvalds's avatar
Linus Torvalds committed
392
393
394
395
396
397
398
399
400
/*
 * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 * offset, we return EINVAL.
 */
sys_mmap2:
#if PAGE_SHIFT > 12
		tst	r5, #PGOFF_MASK
		moveq	r5, r5, lsr #PAGE_SHIFT - 12
		streq	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
401
		beq	sys_mmap_pgoff
Linus Torvalds's avatar
Linus Torvalds committed
402
		mov	r0, #-EINVAL
403
		ret	lr
Linus Torvalds's avatar
Linus Torvalds committed
404
405
#else
		str	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
406
		b	sys_mmap_pgoff
Linus Torvalds's avatar
Linus Torvalds committed
407
#endif
408
ENDPROC(sys_mmap2)
409
410

#ifdef CONFIG_OABI_COMPAT
411

412
413
414
415
416
417
418
/*
 * These are syscalls with argument register differences
 */

sys_oabi_pread64:
		stmia	sp, {r3, r4}
		b	sys_pread64
419
ENDPROC(sys_oabi_pread64)
420
421
422
423

sys_oabi_pwrite64:
		stmia	sp, {r3, r4}
		b	sys_pwrite64
424
ENDPROC(sys_oabi_pwrite64)
425
426
427
428
429

sys_oabi_truncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_truncate64
430
ENDPROC(sys_oabi_truncate64)
431
432
433
434
435

sys_oabi_ftruncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_ftruncate64
436
ENDPROC(sys_oabi_ftruncate64)
437
438
439
440
441
442

sys_oabi_readahead:
		str	r3, [sp]
		mov	r3, r2
		mov	r2, r1
		b	sys_readahead
443
ENDPROC(sys_oabi_readahead)
444

445
446
447
448
/*
 * Let's declare a second syscall table for old ABI binaries
 * using the compatibility syscall entries.
 */
449
450
451
452
	syscall_table_start sys_oabi_call_table
#define COMPAT(nr, native, compat) syscall nr, compat
#include <calls-oabi.S>
	syscall_table_end sys_oabi_call_table
453

454
455
#endif