core.c 27.8 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * core.c - Kernel Live Patching Core
 *
 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
 * Copyright (C) 2014 SUSE
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/kallsyms.h>
#include <linux/livepatch.h>
18
19
#include <linux/elf.h>
#include <linux/moduleloader.h>
20
#include <linux/completion.h>
21
#include <linux/memory.h>
22
#include <asm/cacheflush.h>
23
#include "core.h"
24
#include "patch.h"
25
#include "transition.h"
26

27
/*
28
29
30
31
32
33
 * klp_mutex is a coarse lock which serializes access to klp data.  All
 * accesses to klp-related variables and structures must have mutex protection,
 * except within the following functions which carefully avoid the need for it:
 *
 * - klp_ftrace_handler()
 * - klp_update_patch_state()
34
 */
35
DEFINE_MUTEX(klp_mutex);
36

37
38
39
40
41
/*
 * Actively used patches: enabled or in transition. Note that replaced
 * or disabled patches are not listed even though the related kernel
 * module still can be loaded.
 */
42
LIST_HEAD(klp_patches);
43
44
45
46
47
48
49
50
51
52
53

static struct kobject *klp_root_kobj;

static bool klp_is_module(struct klp_object *obj)
{
	return obj->name;
}

/* sets obj->mod if object is not vmlinux and module is found */
static void klp_find_object_module(struct klp_object *obj)
{
54
55
	struct module *mod;

56
57
58
59
60
	if (!klp_is_module(obj))
		return;

	mutex_lock(&module_mutex);
	/*
61
62
	 * We do not want to block removal of patched modules and therefore
	 * we do not take a reference here. The patches are removed by
63
	 * klp_module_going() instead.
64
65
66
	 */
	mod = find_module(obj->name);
	/*
67
68
	 * Do not mess work of klp_module_coming() and klp_module_going().
	 * Note that the patch might still be needed before klp_module_going()
69
70
71
	 * is called. Module functions can be called even in the GOING state
	 * until mod->exit() finishes. This is especially important for
	 * patches that modify semantic of the functions.
72
	 */
73
74
75
	if (mod && mod->klp_alive)
		obj->mod = mod;

76
77
78
79
80
	mutex_unlock(&module_mutex);
}

static bool klp_initialized(void)
{
81
	return !!klp_root_kobj;
82
83
}

Jason Baron's avatar
Jason Baron committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
static struct klp_func *klp_find_func(struct klp_object *obj,
				      struct klp_func *old_func)
{
	struct klp_func *func;

	klp_for_each_func(obj, func) {
		if ((strcmp(old_func->old_name, func->old_name) == 0) &&
		    (old_func->old_sympos == func->old_sympos)) {
			return func;
		}
	}

	return NULL;
}

static struct klp_object *klp_find_object(struct klp_patch *patch,
					  struct klp_object *old_obj)
{
	struct klp_object *obj;

	klp_for_each_object(patch, obj) {
		if (klp_is_module(old_obj)) {
			if (klp_is_module(obj) &&
			    strcmp(old_obj->name, obj->name) == 0) {
				return obj;
			}
		} else if (!klp_is_module(obj)) {
			return obj;
		}
	}

	return NULL;
}

118
119
120
121
122
struct klp_find_arg {
	const char *objname;
	const char *name;
	unsigned long addr;
	unsigned long count;
123
	unsigned long pos;
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
};

static int klp_find_callback(void *data, const char *name,
			     struct module *mod, unsigned long addr)
{
	struct klp_find_arg *args = data;

	if ((mod && !args->objname) || (!mod && args->objname))
		return 0;

	if (strcmp(args->name, name))
		return 0;

	if (args->objname && strcmp(args->objname, mod->name))
		return 0;

	args->addr = addr;
	args->count++;

143
144
145
146
147
148
149
150
	/*
	 * Finish the search when the symbol is found for the desired position
	 * or the position is not defined for a non-unique symbol.
	 */
	if ((args->pos && (args->count == args->pos)) ||
	    (!args->pos && (args->count > 1)))
		return 1;

151
152
153
154
	return 0;
}

static int klp_find_object_symbol(const char *objname, const char *name,
155
				  unsigned long sympos, unsigned long *addr)
156
157
158
159
160
{
	struct klp_find_arg args = {
		.objname = objname,
		.name = name,
		.addr = 0,
161
162
		.count = 0,
		.pos = sympos,
163
164
	};

165
	mutex_lock(&module_mutex);
166
167
168
169
	if (objname)
		module_kallsyms_on_each_symbol(klp_find_callback, &args);
	else
		kallsyms_on_each_symbol(klp_find_callback, &args);
170
	mutex_unlock(&module_mutex);
171

172
173
174
175
176
	/*
	 * Ensure an address was found. If sympos is 0, ensure symbol is unique;
	 * otherwise ensure the symbol position count matches sympos.
	 */
	if (args.addr == 0)
177
		pr_err("symbol '%s' not found in symbol table\n", name);
178
	else if (args.count > 1 && sympos == 0) {
179
180
		pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
		       name, objname);
181
182
183
184
	} else if (sympos != args.count && sympos > 0) {
		pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
		       sympos, name, objname ? objname : "vmlinux");
	} else {
185
186
187
188
189
190
191
192
		*addr = args.addr;
		return 0;
	}

	*addr = 0;
	return -EINVAL;
}

193
static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
194
{
195
196
197
198
199
200
201
	int i, cnt, vmlinux, ret;
	char objname[MODULE_NAME_LEN];
	char symname[KSYM_NAME_LEN];
	char *strtab = pmod->core_kallsyms.strtab;
	Elf_Rela *relas;
	Elf_Sym *sym;
	unsigned long sympos, addr;
202

203
	/*
204
205
206
207
208
209
210
211
	 * Since the field widths for objname and symname in the sscanf()
	 * call are hard-coded and correspond to MODULE_NAME_LEN and
	 * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
	 * and KSYM_NAME_LEN have the values we expect them to have.
	 *
	 * Because the value of MODULE_NAME_LEN can differ among architectures,
	 * we use the smallest/strictest upper bound possible (56, based on
	 * the current definition of MODULE_NAME_LEN) to prevent overflows.
212
	 */
213
214
215
216
217
218
219
	BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);

	relas = (Elf_Rela *) relasec->sh_addr;
	/* For each rela in this klp relocation section */
	for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
		sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
		if (sym->st_shndx != SHN_LIVEPATCH) {
220
			pr_err("symbol %s is not marked as a livepatch symbol\n",
221
222
223
224
225
226
227
228
229
			       strtab + sym->st_name);
			return -EINVAL;
		}

		/* Format: .klp.sym.objname.symname,sympos */
		cnt = sscanf(strtab + sym->st_name,
			     ".klp.sym.%55[^.].%127[^,],%lu",
			     objname, symname, &sympos);
		if (cnt != 3) {
230
			pr_err("symbol %s has an incorrectly formatted name\n",
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
			       strtab + sym->st_name);
			return -EINVAL;
		}

		/* klp_find_object_symbol() treats a NULL objname as vmlinux */
		vmlinux = !strcmp(objname, "vmlinux");
		ret = klp_find_object_symbol(vmlinux ? NULL : objname,
					     symname, sympos, &addr);
		if (ret)
			return ret;

		sym->st_value = addr;
	}

	return 0;
246
247
248
249
250
}

static int klp_write_object_relocations(struct module *pmod,
					struct klp_object *obj)
{
251
252
253
254
	int i, cnt, ret = 0;
	const char *objname, *secname;
	char sec_objname[MODULE_NAME_LEN];
	Elf_Shdr *sec;
255
256
257
258

	if (WARN_ON(!klp_is_object_loaded(obj)))
		return -EINVAL;

259
	objname = klp_is_module(obj) ? obj->name : "vmlinux";
260

261
262
263
264
265
266
	/* For each klp relocation section */
	for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
		sec = pmod->klp_info->sechdrs + i;
		secname = pmod->klp_info->secstrings + sec->sh_name;
		if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
			continue;
267

268
269
270
271
272
273
274
		/*
		 * Format: .klp.rela.sec_objname.section_name
		 * See comment in klp_resolve_symbols() for an explanation
		 * of the selected field width value.
		 */
		cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
		if (cnt != 1) {
275
			pr_err("section %s has an incorrectly formatted name\n",
276
277
278
279
			       secname);
			ret = -EINVAL;
			break;
		}
280

281
282
		if (strcmp(objname, sec_objname))
			continue;
283

284
		ret = klp_resolve_symbols(sec, pmod);
285
		if (ret)
286
			break;
287

288
289
290
291
292
		ret = apply_relocate_add(pmod->klp_info->sechdrs,
					 pmod->core_kallsyms.strtab,
					 pmod->klp_info->symndx, i, pmod);
		if (ret)
			break;
293
294
	}

295
	return ret;
296
297
298
299
300
301
302
303
}

/*
 * Sysfs Interface
 *
 * /sys/kernel/livepatch
 * /sys/kernel/livepatch/<patch>
 * /sys/kernel/livepatch/<patch>/enabled
304
 * /sys/kernel/livepatch/<patch>/transition
305
 * /sys/kernel/livepatch/<patch>/force
306
 * /sys/kernel/livepatch/<patch>/<object>
307
 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
308
 */
309
static int __klp_disable_patch(struct klp_patch *patch);
310
311
312
313
314
315

static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
			     const char *buf, size_t count)
{
	struct klp_patch *patch;
	int ret;
316
	bool enabled;
317

318
	ret = kstrtobool(buf, &enabled);
319
	if (ret)
320
		return ret;
321
322
323
324
325

	patch = container_of(kobj, struct klp_patch, kobj);

	mutex_lock(&klp_mutex);

326
	if (patch->enabled == enabled) {
327
328
		/* already in requested state */
		ret = -EINVAL;
329
		goto out;
330
331
	}

332
333
334
335
336
337
338
339
	/*
	 * Allow to reverse a pending transition in both ways. It might be
	 * necessary to complete the transition without forcing and breaking
	 * the system integrity.
	 *
	 * Do not allow to re-enable a disabled patch.
	 */
	if (patch == klp_transition_patch)
340
		klp_reverse_transition();
341
	else if (!enabled)
342
		ret = __klp_disable_patch(patch);
343
344
	else
		ret = -EINVAL;
345

346
out:
347
348
	mutex_unlock(&klp_mutex);

349
350
	if (ret)
		return ret;
351
352
353
354
355
356
357
358
359
	return count;
}

static ssize_t enabled_show(struct kobject *kobj,
			    struct kobj_attribute *attr, char *buf)
{
	struct klp_patch *patch;

	patch = container_of(kobj, struct klp_patch, kobj);
360
	return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
361
362
}

363
364
365
366
367
368
369
370
static ssize_t transition_show(struct kobject *kobj,
			       struct kobj_attribute *attr, char *buf)
{
	struct klp_patch *patch;

	patch = container_of(kobj, struct klp_patch, kobj);
	return snprintf(buf, PAGE_SIZE-1, "%d\n",
			patch == klp_transition_patch);
371
372
}

373
374
375
376
377
378
379
380
381
382
383
static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr,
			   const char *buf, size_t count)
{
	struct klp_patch *patch;
	int ret;
	bool val;

	ret = kstrtobool(buf, &val);
	if (ret)
		return ret;

384
385
386
387
388
389
390
391
392
393
394
395
396
397
	if (!val)
		return count;

	mutex_lock(&klp_mutex);

	patch = container_of(kobj, struct klp_patch, kobj);
	if (patch != klp_transition_patch) {
		mutex_unlock(&klp_mutex);
		return -EINVAL;
	}

	klp_force_transition();

	mutex_unlock(&klp_mutex);
398
399
400
401

	return count;
}

402
static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
403
static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
404
static struct kobj_attribute force_kobj_attr = __ATTR_WO(force);
405
406
static struct attribute *klp_patch_attrs[] = {
	&enabled_kobj_attr.attr,
407
	&transition_kobj_attr.attr,
408
	&force_kobj_attr.attr,
409
410
	NULL
};
411
ATTRIBUTE_GROUPS(klp_patch);
412

Jason Baron's avatar
Jason Baron committed
413
414
415
416
417
418
static void klp_free_object_dynamic(struct klp_object *obj)
{
	kfree(obj->name);
	kfree(obj);
}

419
420
421
422
static void klp_init_func_early(struct klp_object *obj,
				struct klp_func *func);
static void klp_init_object_early(struct klp_patch *patch,
				  struct klp_object *obj);
423

424
425
static struct klp_object *klp_alloc_object_dynamic(const char *name,
						   struct klp_patch *patch)
Jason Baron's avatar
Jason Baron committed
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
{
	struct klp_object *obj;

	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
	if (!obj)
		return NULL;

	if (name) {
		obj->name = kstrdup(name, GFP_KERNEL);
		if (!obj->name) {
			kfree(obj);
			return NULL;
		}
	}

441
	klp_init_object_early(patch, obj);
Jason Baron's avatar
Jason Baron committed
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
	obj->dynamic = true;

	return obj;
}

static void klp_free_func_nop(struct klp_func *func)
{
	kfree(func->old_name);
	kfree(func);
}

static struct klp_func *klp_alloc_func_nop(struct klp_func *old_func,
					   struct klp_object *obj)
{
	struct klp_func *func;

	func = kzalloc(sizeof(*func), GFP_KERNEL);
	if (!func)
		return NULL;

	if (old_func->old_name) {
		func->old_name = kstrdup(old_func->old_name, GFP_KERNEL);
		if (!func->old_name) {
			kfree(func);
			return NULL;
		}
	}

470
	klp_init_func_early(obj, func);
Jason Baron's avatar
Jason Baron committed
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
	/*
	 * func->new_func is same as func->old_func. These addresses are
	 * set when the object is loaded, see klp_init_object_loaded().
	 */
	func->old_sympos = old_func->old_sympos;
	func->nop = true;

	return func;
}

static int klp_add_object_nops(struct klp_patch *patch,
			       struct klp_object *old_obj)
{
	struct klp_object *obj;
	struct klp_func *func, *old_func;

	obj = klp_find_object(patch, old_obj);

	if (!obj) {
490
		obj = klp_alloc_object_dynamic(old_obj->name, patch);
Jason Baron's avatar
Jason Baron committed
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
		if (!obj)
			return -ENOMEM;
	}

	klp_for_each_func(old_obj, old_func) {
		func = klp_find_func(obj, old_func);
		if (func)
			continue;

		func = klp_alloc_func_nop(old_func, obj);
		if (!func)
			return -ENOMEM;
	}

	return 0;
}

/*
 * Add 'nop' functions which simply return to the caller to run
 * the original function. The 'nop' functions are added to a
 * patch to facilitate a 'replace' mode.
 */
static int klp_add_nops(struct klp_patch *patch)
{
	struct klp_patch *old_patch;
	struct klp_object *old_obj;

518
	klp_for_each_patch(old_patch) {
Jason Baron's avatar
Jason Baron committed
519
520
521
522
523
524
525
526
527
528
529
530
		klp_for_each_object(old_patch, old_obj) {
			int err;

			err = klp_add_object_nops(patch, old_obj);
			if (err)
				return err;
		}
	}

	return 0;
}

531
532
static void klp_kobj_release_patch(struct kobject *kobj)
{
533
534
535
536
	struct klp_patch *patch;

	patch = container_of(kobj, struct klp_patch, kobj);
	complete(&patch->finish);
537
538
539
540
541
}

static struct kobj_type klp_ktype_patch = {
	.release = klp_kobj_release_patch,
	.sysfs_ops = &kobj_sysfs_ops,
542
	.default_groups = klp_patch_groups,
543
544
};

545
546
static void klp_kobj_release_object(struct kobject *kobj)
{
Jason Baron's avatar
Jason Baron committed
547
548
549
550
551
552
	struct klp_object *obj;

	obj = container_of(kobj, struct klp_object, kobj);

	if (obj->dynamic)
		klp_free_object_dynamic(obj);
553
554
555
556
557
558
559
}

static struct kobj_type klp_ktype_object = {
	.release = klp_kobj_release_object,
	.sysfs_ops = &kobj_sysfs_ops,
};

560
561
static void klp_kobj_release_func(struct kobject *kobj)
{
Jason Baron's avatar
Jason Baron committed
562
563
564
565
566
567
	struct klp_func *func;

	func = container_of(kobj, struct klp_func, kobj);

	if (func->nop)
		klp_free_func_nop(func);
568
569
570
571
572
573
574
}

static struct kobj_type klp_ktype_func = {
	.release = klp_kobj_release_func,
	.sysfs_ops = &kobj_sysfs_ops,
};

575
static void __klp_free_funcs(struct klp_object *obj, bool nops_only)
576
{
Jason Baron's avatar
Jason Baron committed
577
	struct klp_func *func, *tmp_func;
578

Jason Baron's avatar
Jason Baron committed
579
	klp_for_each_func_safe(obj, func, tmp_func) {
580
581
582
583
		if (nops_only && !func->nop)
			continue;

		list_del(&func->node);
584
		kobject_put(&func->kobj);
585
	}
586
587
588
589
590
591
592
593
594
}

/* Clean up when a patched object is unloaded */
static void klp_free_object_loaded(struct klp_object *obj)
{
	struct klp_func *func;

	obj->mod = NULL;

Jason Baron's avatar
Jason Baron committed
595
	klp_for_each_func(obj, func) {
596
		func->old_func = NULL;
Jason Baron's avatar
Jason Baron committed
597
598
599
600

		if (func->nop)
			func->new_func = NULL;
	}
601
602
}

603
static void __klp_free_objects(struct klp_patch *patch, bool nops_only)
604
{
Jason Baron's avatar
Jason Baron committed
605
	struct klp_object *obj, *tmp_obj;
606

Jason Baron's avatar
Jason Baron committed
607
	klp_for_each_object_safe(patch, obj, tmp_obj) {
608
609
610
611
612
613
		__klp_free_funcs(obj, nops_only);

		if (nops_only && !obj->dynamic)
			continue;

		list_del(&obj->node);
614
		kobject_put(&obj->kobj);
615
616
617
	}
}

618
619
620
621
622
623
624
625
626
627
static void klp_free_objects(struct klp_patch *patch)
{
	__klp_free_objects(patch, false);
}

static void klp_free_objects_dynamic(struct klp_patch *patch)
{
	__klp_free_objects(patch, true);
}

628
629
630
631
632
633
634
/*
 * This function implements the free operations that can be called safely
 * under klp_mutex.
 *
 * The operation must be completed by calling klp_free_patch_finish()
 * outside klp_mutex.
 */
635
void klp_free_patch_start(struct klp_patch *patch)
636
637
638
{
	if (!list_empty(&patch->list))
		list_del(&patch->list);
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658

	klp_free_objects(patch);
}

/*
 * This function implements the free part that must be called outside
 * klp_mutex.
 *
 * It must be called after klp_free_patch_start(). And it has to be
 * the last function accessing the livepatch structures when the patch
 * gets disabled.
 */
static void klp_free_patch_finish(struct klp_patch *patch)
{
	/*
	 * Avoid deadlock with enabled_store() sysfs callback by
	 * calling this outside klp_mutex. It is safe because
	 * this is called when the patch gets disabled and it
	 * cannot get enabled again.
	 */
659
660
	kobject_put(&patch->kobj);
	wait_for_completion(&patch->finish);
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677

	/* Put the module after the last access to struct klp_patch. */
	if (!patch->forced)
		module_put(patch->mod);
}

/*
 * The livepatch might be freed from sysfs interface created by the patch.
 * This work allows to wait until the interface is destroyed in a separate
 * context.
 */
static void klp_free_patch_work_fn(struct work_struct *work)
{
	struct klp_patch *patch =
		container_of(work, struct klp_patch, free_work);

	klp_free_patch_finish(patch);
678
679
680
681
}

static int klp_init_func(struct klp_object *obj, struct klp_func *func)
{
Jason Baron's avatar
Jason Baron committed
682
683
684
685
686
687
688
689
	if (!func->old_name)
		return -EINVAL;

	/*
	 * NOPs get the address later. The patched module must be loaded,
	 * see klp_init_object_loaded().
	 */
	if (!func->new_func && !func->nop)
690
691
		return -EINVAL;

692
693
694
	if (strlen(func->old_name) >= KSYM_NAME_LEN)
		return -EINVAL;

695
	INIT_LIST_HEAD(&func->stack_node);
696
	func->patched = false;
697
	func->transition = false;
698

699
700
701
702
703
	/* The format for the sysfs directory is <function,sympos> where sympos
	 * is the nth occurrence of this symbol in kallsyms for the patched
	 * object. If the user selects 0 for old_sympos, then 1 will be used
	 * since a unique symbol will be the first occurrence.
	 */
704
705
706
	return kobject_add(&func->kobj, &obj->kobj, "%s,%lu",
			   func->old_name,
			   func->old_sympos ? func->old_sympos : 1);
707
708
}

709
710
711
712
713
714
/* Arches may override this to finish any remaining arch-specific tasks */
void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
					struct klp_object *obj)
{
}

715
716
717
718
719
720
721
/* parts of the initialization that is done only when the object is loaded */
static int klp_init_object_loaded(struct klp_patch *patch,
				  struct klp_object *obj)
{
	struct klp_func *func;
	int ret;

722
723
	mutex_lock(&text_mutex);

724
	module_disable_ro(patch->mod);
725
	ret = klp_write_object_relocations(patch->mod, obj);
726
727
	if (ret) {
		module_enable_ro(patch->mod, true);
728
		mutex_unlock(&text_mutex);
729
		return ret;
730
731
732
733
	}

	arch_klp_init_object_loaded(patch, obj);
	module_enable_ro(patch->mod, true);
734

735
736
	mutex_unlock(&text_mutex);

737
	klp_for_each_func(obj, func) {
738
739
		ret = klp_find_object_symbol(obj->name, func->old_name,
					     func->old_sympos,
740
					     (unsigned long *)&func->old_func);
741
742
		if (ret)
			return ret;
743

744
		ret = kallsyms_lookup_size_offset((unsigned long)func->old_func,
745
746
747
748
749
750
751
						  &func->old_size, NULL);
		if (!ret) {
			pr_err("kallsyms size lookup failed for '%s'\n",
			       func->old_name);
			return -ENOENT;
		}

Jason Baron's avatar
Jason Baron committed
752
753
754
		if (func->nop)
			func->new_func = func->old_func;

755
756
757
758
759
760
761
		ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
						  &func->new_size, NULL);
		if (!ret) {
			pr_err("kallsyms size lookup failed for '%s' replacement\n",
			       func->old_name);
			return -ENOENT;
		}
762
763
764
765
766
767
768
769
770
771
772
	}

	return 0;
}

static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
{
	struct klp_func *func;
	int ret;
	const char *name;

773
774
775
	if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN)
		return -EINVAL;

776
	obj->patched = false;
777
	obj->mod = NULL;
778
779
780
781

	klp_find_object_module(obj);

	name = klp_is_module(obj) ? obj->name : "vmlinux";
782
	ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name);
783
784
	if (ret)
		return ret;
785

786
	klp_for_each_func(obj, func) {
787
788
		ret = klp_init_func(obj, func);
		if (ret)
789
			return ret;
790
791
	}

792
	if (klp_is_object_loaded(obj))
793
794
795
796
797
		ret = klp_init_object_loaded(patch, obj);

	return ret;
}

798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
static void klp_init_func_early(struct klp_object *obj,
				struct klp_func *func)
{
	kobject_init(&func->kobj, &klp_ktype_func);
	list_add_tail(&func->node, &obj->func_list);
}

static void klp_init_object_early(struct klp_patch *patch,
				  struct klp_object *obj)
{
	INIT_LIST_HEAD(&obj->func_list);
	kobject_init(&obj->kobj, &klp_ktype_object);
	list_add_tail(&obj->node, &patch->obj_list);
}

813
static int klp_init_patch_early(struct klp_patch *patch)
814
815
{
	struct klp_object *obj;
816
	struct klp_func *func;
817
818
819
820

	if (!patch->objs)
		return -EINVAL;

821
	INIT_LIST_HEAD(&patch->list);
822
	INIT_LIST_HEAD(&patch->obj_list);
823
	kobject_init(&patch->kobj, &klp_ktype_patch);
824
	patch->enabled = false;
825
	patch->forced = false;
826
	INIT_WORK(&patch->free_work, klp_free_patch_work_fn);
827
	init_completion(&patch->finish);
828

829
	klp_for_each_object_static(patch, obj) {
830
831
832
		if (!obj->funcs)
			return -EINVAL;

833
		klp_init_object_early(patch, obj);
834

835
		klp_for_each_func_static(obj, func) {
836
			klp_init_func_early(obj, func);
837
		}
838
839
	}

840
841
842
	if (!try_module_get(patch->mod))
		return -ENODEV;

843
844
845
846
847
848
849
850
	return 0;
}

static int klp_init_patch(struct klp_patch *patch)
{
	struct klp_object *obj;
	int ret;

851
	ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name);
852
	if (ret)
853
		return ret;
854

Jason Baron's avatar
Jason Baron committed
855
856
857
858
859
860
	if (patch->replace) {
		ret = klp_add_nops(patch);
		if (ret)
			return ret;
	}

861
	klp_for_each_object(patch, obj) {
862
863
		ret = klp_init_object(patch, obj);
		if (ret)
864
			return ret;
865
866
	}

867
	list_add_tail(&patch->list, &klp_patches);
868

869
	return 0;
870
871
}

872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
static int __klp_disable_patch(struct klp_patch *patch)
{
	struct klp_object *obj;

	if (WARN_ON(!patch->enabled))
		return -EINVAL;

	if (klp_transition_patch)
		return -EBUSY;

	klp_init_transition(patch, KLP_UNPATCHED);

	klp_for_each_object(patch, obj)
		if (obj->patched)
			klp_pre_unpatch_callback(obj);

	/*
	 * Enforce the order of the func->transition writes in
	 * klp_init_transition() and the TIF_PATCH_PENDING writes in
	 * klp_start_transition().  In the rare case where klp_ftrace_handler()
	 * is called shortly after klp_update_patch_state() switches the task,
	 * this ensures the handler sees that func->transition is set.
	 */
	smp_wmb();

	klp_start_transition();
	patch->enabled = false;
899
	klp_try_complete_transition();
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948

	return 0;
}

static int __klp_enable_patch(struct klp_patch *patch)
{
	struct klp_object *obj;
	int ret;

	if (klp_transition_patch)
		return -EBUSY;

	if (WARN_ON(patch->enabled))
		return -EINVAL;

	pr_notice("enabling patch '%s'\n", patch->mod->name);

	klp_init_transition(patch, KLP_PATCHED);

	/*
	 * Enforce the order of the func->transition writes in
	 * klp_init_transition() and the ops->func_stack writes in
	 * klp_patch_object(), so that klp_ftrace_handler() will see the
	 * func->transition updates before the handler is registered and the
	 * new funcs become visible to the handler.
	 */
	smp_wmb();

	klp_for_each_object(patch, obj) {
		if (!klp_is_object_loaded(obj))
			continue;

		ret = klp_pre_patch_callback(obj);
		if (ret) {
			pr_warn("pre-patch callback failed for object '%s'\n",
				klp_is_module(obj) ? obj->name : "vmlinux");
			goto err;
		}

		ret = klp_patch_object(obj);
		if (ret) {
			pr_warn("failed to patch object '%s'\n",
				klp_is_module(obj) ? obj->name : "vmlinux");
			goto err;
		}
	}

	klp_start_transition();
	patch->enabled = true;
949
	klp_try_complete_transition();
950
951
952
953
954
955
956
957
958
959

	return 0;
err:
	pr_warn("failed to enable patch '%s'\n", patch->mod->name);

	klp_cancel_transition();
	return ret;
}

/**
960
961
 * klp_enable_patch() - enable the livepatch
 * @patch:	patch to be enabled
962
 *
963
964
965
966
967
968
 * Initializes the data structure associated with the patch, creates the sysfs
 * interface, performs the needed symbol lookups and code relocations,
 * registers the patched functions with ftrace.
 *
 * This function is supposed to be called from the livepatch module_init()
 * callback.
969
970
971
972
973
974
975
 *
 * Return: 0 on success, otherwise error
 */
int klp_enable_patch(struct klp_patch *patch)
{
	int ret;

976
977
978
979
980
981
982
983
984
985
986
987
988
	if (!patch || !patch->mod)
		return -EINVAL;

	if (!is_livepatch_module(patch->mod)) {
		pr_err("module %s is not marked as a livepatch module\n",
		       patch->mod->name);
		return -EINVAL;
	}

	if (!klp_initialized())
		return -ENODEV;

	if (!klp_have_reliable_stack()) {
989
990
		pr_warn("This architecture doesn't have support for the livepatch consistency model.\n");
		pr_warn("The livepatch transition may never complete.\n");
991
992
	}

993
994
	mutex_lock(&klp_mutex);

995
996
997
998
	ret = klp_init_patch_early(patch);
	if (ret) {
		mutex_unlock(&klp_mutex);
		return ret;
999
1000
	}