entry-common.S 10.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
/*
 *  linux/arch/arm/kernel/entry-common.S
 *
 *  Copyright (C) 2000 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

11
#include <asm/assembler.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <asm/unistd.h>
13
#include <asm/ftrace.h>
14
#include <asm/unwind.h>
Linus Torvalds's avatar
Linus Torvalds committed
15

16
17
18
19
20
21
22
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
	.macro  arch_ret_to_user, tmp1, tmp2
	.endm
#endif

Linus Torvalds's avatar
Linus Torvalds committed
23
24
25
26
#include "entry-header.S"


	.align	5
Russell King's avatar
Russell King committed
27
#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING))
Linus Torvalds's avatar
Linus Torvalds committed
28
/*
Russell King's avatar
Russell King committed
29
30
31
32
 * This is the fast syscall return path.  We do as little as possible here,
 * such as avoiding writing r0 to the stack.  We only use this path if we
 * have tracing and context tracking disabled - the overheads from those
 * features make this path too inefficient.
Linus Torvalds's avatar
Linus Torvalds committed
33
34
 */
ret_fast_syscall:
35
36
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
Russell King's avatar
Russell King committed
37
	disable_irq_notrace			@ disable interrupts
38
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
Russell King's avatar
Russell King committed
39
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Linus Torvalds's avatar
Linus Torvalds committed
40
	bne	fast_work_pending
41

42
43
44
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr

45
	restore_user_regs fast = 1, offset = S_OFF
46
 UNWIND(.fnend		)
Russell King's avatar
Russell King committed
47
ENDPROC(ret_fast_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
48

Russell King's avatar
Russell King committed
49
	/* Ok, we need to do extra processing, enter the slow path. */
Linus Torvalds's avatar
Linus Torvalds committed
50
51
fast_work_pending:
	str	r0, [sp, #S_R0+S_OFF]!		@ returned r0
Russell King's avatar
Russell King committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
	/* fall through to work_pending */
#else
/*
 * The "replacement" ret_fast_syscall for when tracing or context tracking
 * is enabled.  As we will need to call out to some C functions, we save
 * r0 first to avoid needing to save registers around each C function call.
 */
ret_fast_syscall:
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
	disable_irq_notrace			@ disable interrupts
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
	beq	no_work_pending
 UNWIND(.fnend		)
ENDPROC(ret_fast_syscall)

	/* Slower path - fall through to work_pending */
#endif

	tst	r1, #_TIF_SYSCALL_WORK
	bne	__sys_trace_return_nosave
slow_work_pending:
Linus Torvalds's avatar
Linus Torvalds committed
76
77
	mov	r0, sp				@ 'regs'
	mov	r2, why				@ 'syscall'
78
	bl	do_work_pending
79
	cmp	r0, #0
80
	beq	no_work_pending
81
	movlt	scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
82
83
84
	ldmia	sp, {r0 - r6}			@ have to reload r0 - r6
	b	local_restart			@ ... and off we go

Linus Torvalds's avatar
Linus Torvalds committed
85
86
/*
 * "slow" syscall return path.  "why" tells us if this was a real syscall.
Russell King's avatar
Russell King committed
87
88
89
 * IRQs may be enabled here, so always disable them.  Note that we use the
 * "notrace" version to avoid calling into the tracing code unnecessarily.
 * do_work_pending() will update this state if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
90
91
92
 */
ENTRY(ret_to_user)
ret_slow_syscall:
Russell King's avatar
Russell King committed
93
	disable_irq_notrace			@ disable interrupts
94
ENTRY(ret_to_user_from_irq)
Linus Torvalds's avatar
Linus Torvalds committed
95
96
	ldr	r1, [tsk, #TI_FLAGS]
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
97
	bne	slow_work_pending
Linus Torvalds's avatar
Linus Torvalds committed
98
no_work_pending:
Russell King's avatar
Russell King committed
99
	asm_trace_hardirqs_on save = 0
100

101
102
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr
103
	ct_user_enter save = 0
104

105
	restore_user_regs fast = 0, offset = 0
106
ENDPROC(ret_to_user_from_irq)
107
ENDPROC(ret_to_user)
Linus Torvalds's avatar
Linus Torvalds committed
108
109
110
111
112
113

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
114
115
	cmp	r5, #0
	movne	r0, r4
116
	badrne	lr, 1f
117
	retne	r5
118
1:	get_thread_info tsk
Linus Torvalds's avatar
Linus Torvalds committed
119
	b	ret_slow_syscall
120
ENDPROC(ret_from_fork)
Linus Torvalds's avatar
Linus Torvalds committed
121

122
123
	.equ NR_syscalls,0
#define CALL(x) .equ NR_syscalls,NR_syscalls+1
Linus Torvalds's avatar
Linus Torvalds committed
124
#include "calls.S"
125
126
127
128
129
130
131
132
133

/*
 * Ensure that the system call table is equal to __NR_syscalls,
 * which is the value the rest of the system sees
 */
.ifne NR_syscalls - __NR_syscalls
.error "__NR_syscalls is not equal to the size of the syscall table"
.endif

134
135
#undef CALL
#define CALL(x) .long x
Linus Torvalds's avatar
Linus Torvalds committed
136
137
138
139
140
141
142
143

/*=============================================================================
 * SWI handler
 *-----------------------------------------------------------------------------
 */

	.align	5
ENTRY(vector_swi)
144
145
146
#ifdef CONFIG_CPU_V7M
	v7m_exception_entry
#else
147
148
	sub	sp, sp, #S_FRAME_SIZE
	stmia	sp, {r0 - r12}			@ Calling r0 - r12
149
150
151
152
 ARM(	add	r8, sp, #S_PC		)
 ARM(	stmdb	r8, {sp, lr}^		)	@ Calling sp, lr
 THUMB(	mov	r8, sp			)
 THUMB(	store_user_sp_lr r8, r10, S_SP	)	@ calling sp, lr
153
154
155
156
	mrs	r8, spsr			@ called from non-FIQ mode, so ok.
	str	lr, [sp, #S_PC]			@ Save calling PC
	str	r8, [sp, #S_PSR]		@ Save CPSR
	str	r0, [sp, #S_OLD_R0]		@ Save OLD_R0
157
#endif
Linus Torvalds's avatar
Linus Torvalds committed
158
	zero_fp
159
	alignment_trap r10, ip, __cr_alignment
160
161
162
163
	enable_irq
	ct_user_exit
	get_thread_info tsk

164
165
166
	/*
	 * Get the system call number.
	 */
167

168
#if defined(CONFIG_OABI_COMPAT)
169

170
171
172
173
174
175
176
	/*
	 * If we have CONFIG_OABI_COMPAT then we need to look at the swi
	 * value to determine if it is an EABI or an old ABI call.
	 */
#ifdef CONFIG_ARM_THUMB
	tst	r8, #PSR_T_BIT
	movne	r10, #0				@ no thumb OABI emulation
177
 USER(	ldreq	r10, [lr, #-4]		)	@ get SWI instruction
178
#else
179
 USER(	ldr	r10, [lr, #-4]		)	@ get SWI instruction
180
#endif
181
 ARM_BE8(rev	r10, r10)			@ little endian instruction
182
183
184
185
186
187

#elif defined(CONFIG_AEABI)

	/*
	 * Pure EABI user space always put syscall number into scno (r7).
	 */
188
#elif defined(CONFIG_ARM_THUMB)
189
	/* Legacy ABI only, possibly thumb mode. */
190
191
	tst	r8, #PSR_T_BIT			@ this is SPSR from save_user_regs
	addne	scno, r7, #__NR_SYSCALL_BASE	@ put OS number in
192
 USER(	ldreq	scno, [lr, #-4]		)
193

194
#else
195
	/* Legacy ABI only. */
196
 USER(	ldr	scno, [lr, #-4]		)	@ get SWI instruction
197
#endif
Linus Torvalds's avatar
Linus Torvalds committed
198

199
200
201
202
203
204
205
206
207
208
209
210
211
	adr	tbl, sys_call_table		@ load syscall table pointer

#if defined(CONFIG_OABI_COMPAT)
	/*
	 * If the swi argument is zero, this is an EABI call and we do nothing.
	 *
	 * If this is an old ABI call, get the syscall number into scno and
	 * get the old ABI syscall table address.
	 */
	bics	r10, r10, #0xff000000
	eorne	scno, r10, #__NR_OABI_SYSCALL_BASE
	ldrne	tbl, =sys_oabi_call_table
#elif !defined(CONFIG_AEABI)
Linus Torvalds's avatar
Linus Torvalds committed
212
	bic	scno, scno, #0xff000000		@ mask off SWI op-code
213
	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
214
#endif
215

216
local_restart:
Nicolas Pitre's avatar
Nicolas Pitre committed
217
	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
218
	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
Nicolas Pitre's avatar
Nicolas Pitre committed
219

220
	tst	r10, #_TIF_SYSCALL_WORK		@ are we tracing syscalls?
Linus Torvalds's avatar
Linus Torvalds committed
221
222
223
	bne	__sys_trace

	cmp	scno, #NR_syscalls		@ check upper syscall limit
224
	badr	lr, ret_fast_syscall		@ return address
Linus Torvalds's avatar
Linus Torvalds committed
225
226
227
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine

	add	r1, sp, #S_OFF
228
2:	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
229
	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
230
	bcs	arm_syscall
231
	mov	why, #0				@ no longer a real syscall
Linus Torvalds's avatar
Linus Torvalds committed
232
	b	sys_ni_syscall			@ not private func
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
	/*
	 * We failed to handle a fault trying to access the page
	 * containing the swi instruction, but we're not really in a
	 * position to return -EFAULT. Instead, return back to the
	 * instruction and re-enter the user fault handling path trying
	 * to page it in. This will likely result in sending SEGV to the
	 * current task.
	 */
9001:
	sub	lr, lr, #4
	str	lr, [sp, #S_PC]
	b	ret_fast_syscall
#endif
248
ENDPROC(vector_swi)
Linus Torvalds's avatar
Linus Torvalds committed
249
250
251
252
253
254

	/*
	 * This is the really slow path.  We're going to be doing
	 * context switches, and waiting for our parent to respond.
	 */
__sys_trace:
255
256
257
	mov	r1, scno
	add	r0, sp, #S_OFF
	bl	syscall_trace_enter
Linus Torvalds's avatar
Linus Torvalds committed
258

259
	badr	lr, __sys_trace_return		@ return address
260
	mov	scno, r0			@ syscall number (possibly new)
Linus Torvalds's avatar
Linus Torvalds committed
261
262
	add	r1, sp, #S_R0 + S_OFF		@ pointer to regs
	cmp	scno, #NR_syscalls		@ check upper syscall limit
263
264
	ldmccia	r1, {r0 - r6}			@ have to reload r0 - r6
	stmccia	sp, {r4, r5}			@ and update the stack args
Linus Torvalds's avatar
Linus Torvalds committed
265
	ldrcc	pc, [tbl, scno, lsl #2]		@ call sys_* routine
266
267
268
269
	cmp	scno, #-1			@ skip the syscall?
	bne	2b
	add	sp, sp, #S_OFF			@ restore stack
	b	ret_slow_syscall
Linus Torvalds's avatar
Linus Torvalds committed
270
271
272

__sys_trace_return:
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
273
274
	mov	r0, sp
	bl	syscall_trace_exit
Linus Torvalds's avatar
Linus Torvalds committed
275
276
	b	ret_slow_syscall

Russell King's avatar
Russell King committed
277
278
279
280
281
282
__sys_trace_return_nosave:
	asm_trace_hardirqs_off save=0
	mov	r0, sp
	bl	syscall_trace_exit
	b	ret_slow_syscall

Linus Torvalds's avatar
Linus Torvalds committed
283
284
285
286
287
	.align	5
#ifdef CONFIG_ALIGNMENT_TRAP
	.type	__cr_alignment, #object
__cr_alignment:
	.word	cr_alignment
288
289
290
291
292
293
294
295
296
297
298
299
#endif
	.ltorg

/*
 * This is the syscall table declaration for native ABI syscalls.
 * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 */
#define ABI(native, compat) native
#ifdef CONFIG_AEABI
#define OBSOLETE(syscall) sys_ni_syscall
#else
#define OBSOLETE(syscall) syscall
Linus Torvalds's avatar
Linus Torvalds committed
300
301
302
303
304
#endif

	.type	sys_call_table, #object
ENTRY(sys_call_table)
#include "calls.S"
305
306
#undef ABI
#undef OBSOLETE
Linus Torvalds's avatar
Linus Torvalds committed
307
308
309
310
311

/*============================================================================
 * Special system call wrappers
 */
@ r0 = syscall number
312
@ r8 = syscall table
Linus Torvalds's avatar
Linus Torvalds committed
313
sys_syscall:
314
		bic	scno, r0, #__NR_OABI_SYSCALL_BASE
Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
318
319
320
321
322
323
		cmp	scno, #__NR_syscall - __NR_SYSCALL_BASE
		cmpne	scno, #NR_syscalls	@ check range
		stmloia	sp, {r5, r6}		@ shuffle args
		movlo	r0, r1
		movlo	r1, r2
		movlo	r2, r3
		movlo	r3, r4
		ldrlo	pc, [tbl, scno, lsl #2]
		b	sys_ni_syscall
324
ENDPROC(sys_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
325
326
327

sys_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
328
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
329
		b	sys_sigreturn
330
ENDPROC(sys_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
331
332
333

sys_rt_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
334
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
335
		b	sys_rt_sigreturn
336
ENDPROC(sys_rt_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
337

338
339
340
341
sys_statfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_statfs64
342
ENDPROC(sys_statfs64_wrapper)
343
344
345
346
347

sys_fstatfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_fstatfs64
348
ENDPROC(sys_fstatfs64_wrapper)
349

Linus Torvalds's avatar
Linus Torvalds committed
350
351
352
353
354
355
356
357
358
/*
 * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 * offset, we return EINVAL.
 */
sys_mmap2:
#if PAGE_SHIFT > 12
		tst	r5, #PGOFF_MASK
		moveq	r5, r5, lsr #PAGE_SHIFT - 12
		streq	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
359
		beq	sys_mmap_pgoff
Linus Torvalds's avatar
Linus Torvalds committed
360
		mov	r0, #-EINVAL
361
		ret	lr
Linus Torvalds's avatar
Linus Torvalds committed
362
363
#else
		str	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
364
		b	sys_mmap_pgoff
Linus Torvalds's avatar
Linus Torvalds committed
365
#endif
366
ENDPROC(sys_mmap2)
367
368

#ifdef CONFIG_OABI_COMPAT
369

370
371
372
373
374
375
376
/*
 * These are syscalls with argument register differences
 */

sys_oabi_pread64:
		stmia	sp, {r3, r4}
		b	sys_pread64
377
ENDPROC(sys_oabi_pread64)
378
379
380
381

sys_oabi_pwrite64:
		stmia	sp, {r3, r4}
		b	sys_pwrite64
382
ENDPROC(sys_oabi_pwrite64)
383
384
385
386
387

sys_oabi_truncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_truncate64
388
ENDPROC(sys_oabi_truncate64)
389
390
391
392
393

sys_oabi_ftruncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_ftruncate64
394
ENDPROC(sys_oabi_ftruncate64)
395
396
397
398
399
400

sys_oabi_readahead:
		str	r3, [sp]
		mov	r3, r2
		mov	r2, r1
		b	sys_readahead
401
ENDPROC(sys_oabi_readahead)
402

403
404
405
406
407
408
409
410
411
412
413
414
415
/*
 * Let's declare a second syscall table for old ABI binaries
 * using the compatibility syscall entries.
 */
#define ABI(native, compat) compat
#define OBSOLETE(syscall) syscall

	.type	sys_oabi_call_table, #object
ENTRY(sys_oabi_call_table)
#include "calls.S"
#undef ABI
#undef OBSOLETE

416
417
#endif