fpsimd.c 12.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
 * FP/SIMD context switching and fault handling
 *
 * Copyright (C) 2012 ARM Ltd.
 * Author: Catalin Marinas <catalin.marinas@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/bottom_half.h>
21
#include <linux/cpu.h>
22
#include <linux/cpu_pm.h>
23
24
#include <linux/kernel.h>
#include <linux/init.h>
25
#include <linux/percpu.h>
26
#include <linux/preempt.h>
27
#include <linux/sched/signal.h>
28
29
30
31
#include <linux/signal.h>

#include <asm/fpsimd.h>
#include <asm/cputype.h>
32
#include <asm/simd.h>
33
34
35
36
37
38
39
40

#define FPEXC_IOF	(1 << 0)
#define FPEXC_DZF	(1 << 1)
#define FPEXC_OFF	(1 << 2)
#define FPEXC_UFF	(1 << 3)
#define FPEXC_IXF	(1 << 4)
#define FPEXC_IDF	(1 << 7)

41
42
43
44
45
46
47
48
49
50
/*
 * In order to reduce the number of times the FPSIMD state is needlessly saved
 * and restored, we need to keep track of two things:
 * (a) for each task, we need to remember which CPU was the last one to have
 *     the task's FPSIMD state loaded into its FPSIMD registers;
 * (b) for each CPU, we need to remember which task's userland FPSIMD state has
 *     been loaded into its FPSIMD registers most recently, or whether it has
 *     been used to perform kernel mode NEON in the meantime.
 *
 * For (a), we add a 'cpu' field to struct fpsimd_state, which gets updated to
51
 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
 * address of the userland FPSIMD state of the task that was loaded onto the CPU
 * the most recently, or NULL if kernel mode NEON has been performed after that.
 *
 * With this in place, we no longer have to restore the next FPSIMD state right
 * when switching between tasks. Instead, we can defer this check to userland
 * resume, at which time we verify whether the CPU's fpsimd_last_state and the
 * task's fpsimd_state.cpu are still mutually in sync. If this is the case, we
 * can omit the FPSIMD restore.
 *
 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
 * indicate whether or not the userland FPSIMD state of the current task is
 * present in the registers. The flag is set unless the FPSIMD registers of this
 * CPU currently contain the most recent userland FPSIMD state of the current
 * task.
 *
68
69
70
71
72
73
74
 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
 * save the task's FPSIMD context back to task_struct from softirq context.
 * To prevent this from racing with the manipulation of the task's FPSIMD state
 * from task context and thereby corrupting the state, it is necessary to
 * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
 * flag with local_bh_disable() unless softirqs are already masked.
 *
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 * For a certain task, the sequence may look something like this:
 * - the task gets scheduled in; if both the task's fpsimd_state.cpu field
 *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
 *   variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
 *   cleared, otherwise it is set;
 *
 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
 *   userland FPSIMD state is copied from memory to the registers, the task's
 *   fpsimd_state.cpu field is set to the id of the current CPU, the current
 *   CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
 *   TIF_FOREIGN_FPSTATE flag is cleared;
 *
 * - the task executes an ordinary syscall; upon return to userland, the
 *   TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
 *   restored;
 *
 * - the task executes a syscall which executes some NEON instructions; this is
 *   preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
 *   register contents to memory, clears the fpsimd_last_state per-cpu variable
 *   and sets the TIF_FOREIGN_FPSTATE flag;
 *
 * - the task gets preempted after kernel_neon_end() is called; as we have not
 *   returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
 *   whatever is in the FPSIMD registers is not saved to memory, but discarded.
 */
static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
/*
 * Trapped FP/ASIMD access.
 */
void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
{
	/* TODO: implement lazy context saving/restoring */
	WARN_ON(1);
}

/*
 * Raise a SIGFPE for the current process.
 */
void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
{
	siginfo_t info;
	unsigned int si_code = 0;

	if (esr & FPEXC_IOF)
		si_code = FPE_FLTINV;
	else if (esr & FPEXC_DZF)
		si_code = FPE_FLTDIV;
	else if (esr & FPEXC_OFF)
		si_code = FPE_FLTOVF;
	else if (esr & FPEXC_UFF)
		si_code = FPE_FLTUND;
	else if (esr & FPEXC_IXF)
		si_code = FPE_FLTRES;

	memset(&info, 0, sizeof(info));
	info.si_signo = SIGFPE;
	info.si_code = si_code;
	info.si_addr = (void __user *)instruction_pointer(regs);

	send_sig_info(SIGFPE, &info, current);
}

void fpsimd_thread_switch(struct task_struct *next)
{
140
141
	if (!system_supports_fpsimd())
		return;
142
143
144
145
146
147
	/*
	 * Save the current FPSIMD state to memory, but only if whatever is in
	 * the registers is in fact the most recent userland FPSIMD state of
	 * 'current'.
	 */
	if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
148
		fpsimd_save_state(&current->thread.fpsimd_state);
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167

	if (next->mm) {
		/*
		 * If we are switching to a task whose most recent userland
		 * FPSIMD state is already in the registers of *this* cpu,
		 * we can skip loading the state from memory. Otherwise, set
		 * the TIF_FOREIGN_FPSTATE flag so the state will be loaded
		 * upon the next return to userland.
		 */
		struct fpsimd_state *st = &next->thread.fpsimd_state;

		if (__this_cpu_read(fpsimd_last_state) == st
		    && st->cpu == smp_processor_id())
			clear_ti_thread_flag(task_thread_info(next),
					     TIF_FOREIGN_FPSTATE);
		else
			set_ti_thread_flag(task_thread_info(next),
					   TIF_FOREIGN_FPSTATE);
	}
168
169
170
171
}

void fpsimd_flush_thread(void)
{
172
173
	if (!system_supports_fpsimd())
		return;
174
175
176

	local_bh_disable();

177
	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
178
	fpsimd_flush_task_state(current);
179
	set_thread_flag(TIF_FOREIGN_FPSTATE);
180
181

	local_bh_enable();
182
183
}

184
/*
185
186
 * Save the userland FPSIMD state of 'current' to memory, but only if the state
 * currently held in the registers does in fact belong to 'current'
187
188
189
 */
void fpsimd_preserve_current_state(void)
{
190
191
	if (!system_supports_fpsimd())
		return;
192
193
194

	local_bh_disable();

195
196
	if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
		fpsimd_save_state(&current->thread.fpsimd_state);
197
198

	local_bh_enable();
199
200
201
}

/*
202
203
204
205
206
207
 * Load the userland FPSIMD state of 'current' from memory, but only if the
 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
 * state of 'current'
 */
void fpsimd_restore_current_state(void)
{
208
209
	if (!system_supports_fpsimd())
		return;
210
211
212

	local_bh_disable();

213
214
215
216
	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
		struct fpsimd_state *st = &current->thread.fpsimd_state;

		fpsimd_load_state(st);
217
		__this_cpu_write(fpsimd_last_state, st);
218
219
		st->cpu = smp_processor_id();
	}
220
221

	local_bh_enable();
222
223
224
225
226
227
}

/*
 * Load an updated userland FPSIMD state for 'current' from memory and set the
 * flag that indicates that the FPSIMD register contents are the most recent
 * FPSIMD state of 'current'
228
229
230
 */
void fpsimd_update_current_state(struct fpsimd_state *state)
{
231
232
	if (!system_supports_fpsimd())
		return;
233
234
235

	local_bh_disable();

236
	fpsimd_load_state(state);
237
238
239
	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
		struct fpsimd_state *st = &current->thread.fpsimd_state;

240
		__this_cpu_write(fpsimd_last_state, st);
241
242
		st->cpu = smp_processor_id();
	}
243
244

	local_bh_enable();
245
246
}

247
248
249
250
251
252
253
254
/*
 * Invalidate live CPU copies of task t's FPSIMD state
 */
void fpsimd_flush_task_state(struct task_struct *t)
{
	t->thread.fpsimd_state.cpu = NR_CPUS;
}

255
256
#ifdef CONFIG_KERNEL_MODE_NEON

257
DEFINE_PER_CPU(bool, kernel_neon_busy);
258
EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
259

260
261
262
/*
 * Kernel-side NEON support functions
 */
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277

/*
 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
 * context
 *
 * Must not be called unless may_use_simd() returns true.
 * Task context in the FPSIMD registers is saved back to memory as necessary.
 *
 * A matching call to kernel_neon_end() must be made before returning from the
 * calling context.
 *
 * The caller may freely use the FPSIMD registers until kernel_neon_end() is
 * called.
 */
void kernel_neon_begin(void)
278
{
279
280
	if (WARN_ON(!system_supports_fpsimd()))
		return;
281

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
	BUG_ON(!may_use_simd());

	local_bh_disable();

	__this_cpu_write(kernel_neon_busy, true);

	/* Save unsaved task fpsimd state, if any: */
	if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
		fpsimd_save_state(&current->thread.fpsimd_state);

	/* Invalidate any task state remaining in the fpsimd regs: */
	__this_cpu_write(fpsimd_last_state, NULL);

	preempt_disable();

	local_bh_enable();
298
}
299
EXPORT_SYMBOL(kernel_neon_begin);
300

301
302
303
304
305
306
307
308
309
/*
 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
 *
 * Must be called from a context in which kernel_neon_begin() was previously
 * called, with no call to kernel_neon_end() in the meantime.
 *
 * The caller must not use the FPSIMD registers after this function is called,
 * unless kernel_neon_begin() is called again in the meantime.
 */
310
311
void kernel_neon_end(void)
{
312
313
	bool busy;

314
315
	if (!system_supports_fpsimd())
		return;
316
317
318
319
320

	busy = __this_cpu_xchg(kernel_neon_busy, false);
	WARN_ON(!busy);	/* No matching kernel_neon_begin()? */

	preempt_enable();
321
322
323
}
EXPORT_SYMBOL(kernel_neon_end);

324
325
#ifdef CONFIG_EFI

326
327
static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374

/*
 * EFI runtime services support functions
 *
 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
 * This means that for EFI (and only for EFI), we have to assume that FPSIMD
 * is always used rather than being an optional accelerator.
 *
 * These functions provide the necessary support for ensuring FPSIMD
 * save/restore in the contexts from which EFI is used.
 *
 * Do not use them for any other purpose -- if tempted to do so, you are
 * either doing something wrong or you need to propose some refactoring.
 */

/*
 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
 */
void __efi_fpsimd_begin(void)
{
	if (!system_supports_fpsimd())
		return;

	WARN_ON(preemptible());

	if (may_use_simd())
		kernel_neon_begin();
	else {
		fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
		__this_cpu_write(efi_fpsimd_state_used, true);
	}
}

/*
 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
 */
void __efi_fpsimd_end(void)
{
	if (!system_supports_fpsimd())
		return;

	if (__this_cpu_xchg(efi_fpsimd_state_used, false))
		fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
	else
		kernel_neon_end();
}

375
376
#endif /* CONFIG_EFI */

377
378
#endif /* CONFIG_KERNEL_MODE_NEON */

379
380
381
382
383
384
#ifdef CONFIG_CPU_PM
static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
				  unsigned long cmd, void *v)
{
	switch (cmd) {
	case CPU_PM_ENTER:
385
		if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
386
			fpsimd_save_state(&current->thread.fpsimd_state);
387
		this_cpu_write(fpsimd_last_state, NULL);
388
389
390
		break;
	case CPU_PM_EXIT:
		if (current->mm)
391
			set_thread_flag(TIF_FOREIGN_FPSTATE);
392
393
394
395
396
397
398
399
400
401
402
403
		break;
	case CPU_PM_ENTER_FAILED:
	default:
		return NOTIFY_DONE;
	}
	return NOTIFY_OK;
}

static struct notifier_block fpsimd_cpu_pm_notifier_block = {
	.notifier_call = fpsimd_cpu_pm_notifier,
};

404
static void __init fpsimd_pm_init(void)
405
406
407
408
409
410
411
412
{
	cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
}

#else
static inline void fpsimd_pm_init(void) { }
#endif /* CONFIG_CPU_PM */

413
#ifdef CONFIG_HOTPLUG_CPU
414
static int fpsimd_cpu_dead(unsigned int cpu)
415
{
416
417
	per_cpu(fpsimd_last_state, cpu) = NULL;
	return 0;
418
419
420
421
}

static inline void fpsimd_hotplug_init(void)
{
422
423
	cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
				  NULL, fpsimd_cpu_dead);
424
425
426
427
428
429
}

#else
static inline void fpsimd_hotplug_init(void) { }
#endif

430
431
432
433
434
/*
 * FP/SIMD support code initialisation.
 */
static int __init fpsimd_init(void)
{
435
436
437
438
	if (elf_hwcap & HWCAP_FP) {
		fpsimd_pm_init();
		fpsimd_hotplug_init();
	} else {
439
440
441
		pr_notice("Floating-point is not implemented\n");
	}

442
	if (!(elf_hwcap & HWCAP_ASIMD))
443
		pr_notice("Advanced SIMD is not implemented\n");
444

445
446
	return 0;
}
447
core_initcall(fpsimd_init);