ftrace.c 72.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 William Lee Irwin III
 */

16
17
18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
19
#include <linux/seq_file.h>
20
#include <linux/suspend.h>
21
#include <linux/debugfs.h>
22
#include <linux/hardirq.h>
Ingo Molnar's avatar
Ingo Molnar committed
23
#include <linux/kthread.h>
24
#include <linux/uaccess.h>
Ingo Molnar's avatar
Ingo Molnar committed
25
#include <linux/ftrace.h>
26
#include <linux/sysctl.h>
27
#include <linux/slab.h>
28
#include <linux/ctype.h>
29
#include <linux/list.h>
30
#include <linux/hash.h>
31
#include <linux/rcupdate.h>
32

33
#include <trace/events/sched.h>
34

35
#include <asm/ftrace.h>
36
#include <asm/setup.h>
37

38
#include "trace_output.h"
39
#include "trace_stat.h"
40

41
#define FTRACE_WARN_ON(cond)			\
42
43
44
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
45
			ftrace_kill();		\
46
47
		___r;				\
	})
48
49

#define FTRACE_WARN_ON_ONCE(cond)		\
50
51
52
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
53
			ftrace_kill();		\
54
55
		___r;				\
	})
56

57
58
59
60
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)

61
62
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
63
static int last_ftrace_enabled;
64

65
66
67
/* Quick disabling of function tracer. */
int function_trace_stop;

68
69
70
71
72
73
74
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
	struct list_head list;
	struct pid *pid;
};

75
76
77
78
79
80
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

81
static DEFINE_MUTEX(ftrace_lock);
82

83
84
static struct ftrace_ops ftrace_list_end __read_mostly =
{
85
	.func		= ftrace_stub,
86
87
88
89
};

static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
90
ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
91
ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
92

93
94
95
96
97
98
99
100
101
/*
 * Traverse the ftrace_list, invoking all entries.  The reason that we
 * can use rcu_dereference_raw() is that elements removed from this list
 * are simply leaked, so there is no need to interact with a grace-period
 * mechanism.  The rcu_dereference_raw() calls are needed to handle
 * concurrent insertions into the ftrace_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
Ingo Molnar's avatar
Ingo Molnar committed
102
static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
103
{
104
	struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/
105
106
107

	while (op != &ftrace_list_end) {
		op->func(ip, parent_ip);
108
		op = rcu_dereference_raw(op->next); /*see above*/
109
110
111
	};
}

112
113
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
{
114
	if (!test_tsk_trace_trace(current))
115
116
117
118
119
120
121
122
123
124
125
126
		return;

	ftrace_pid_function(ip, parent_ip);
}

static void set_ftrace_pid_function(ftrace_func_t func)
{
	/* do not set ftrace_pid_function to itself! */
	if (func != ftrace_pid_func)
		ftrace_pid_function = func;
}

127
/**
128
 * clear_ftrace_function - reset the ftrace function
129
 *
130
131
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
132
 */
133
void clear_ftrace_function(void)
134
{
135
	ftrace_trace_function = ftrace_stub;
136
	__ftrace_trace_function = ftrace_stub;
137
	ftrace_pid_function = ftrace_stub;
138
139
}

140
141
142
143
144
145
146
147
148
149
150
151
152
153
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
/*
 * For those archs that do not test ftrace_trace_stop in their
 * mcount call site, we need to do it from C.
 */
static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
{
	if (function_trace_stop)
		return;

	__ftrace_trace_function(ip, parent_ip);
}
#endif

Ingo Molnar's avatar
Ingo Molnar committed
154
static int __register_ftrace_function(struct ftrace_ops *ops)
155
{
156
157
158
159
160
161
162
	ops->next = ftrace_list;
	/*
	 * We are entering ops into the ftrace_list but another
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
	 * the ops pointer included into the ftrace_list.
	 */
163
	rcu_assign_pointer(ftrace_list, ops);
164

165
	if (ftrace_enabled) {
166
167
168
169
170
171
172
		ftrace_func_t func;

		if (ops->next == &ftrace_list_end)
			func = ops->func;
		else
			func = ftrace_list_func;

173
		if (!list_empty(&ftrace_pids)) {
174
175
176
177
			set_ftrace_pid_function(func);
			func = ftrace_pid_func;
		}

178
179
180
181
		/*
		 * For one func, simply call it directly.
		 * For more than one func, call the chain.
		 */
182
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
183
		ftrace_trace_function = func;
184
#else
185
		__ftrace_trace_function = func;
186
187
		ftrace_trace_function = ftrace_test_stop_func;
#endif
188
	}
189

190
191
192
	return 0;
}

Ingo Molnar's avatar
Ingo Molnar committed
193
static int __unregister_ftrace_function(struct ftrace_ops *ops)
194
195
196
197
{
	struct ftrace_ops **p;

	/*
198
199
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
200
201
202
203
	 */
	if (ftrace_list == ops && ops->next == &ftrace_list_end) {
		ftrace_trace_function = ftrace_stub;
		ftrace_list = &ftrace_list_end;
Steven Rostedt's avatar
Steven Rostedt committed
204
		return 0;
205
206
207
208
209
210
	}

	for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
		if (*p == ops)
			break;

Steven Rostedt's avatar
Steven Rostedt committed
211
212
	if (*p != ops)
		return -1;
213
214
215

	*p = (*p)->next;

216
217
	if (ftrace_enabled) {
		/* If we only have one func left, then call that directly */
218
219
220
		if (ftrace_list->next == &ftrace_list_end) {
			ftrace_func_t func = ftrace_list->func;

221
			if (!list_empty(&ftrace_pids)) {
222
223
224
225
226
227
228
229
230
				set_ftrace_pid_function(func);
				func = ftrace_pid_func;
			}
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
			ftrace_trace_function = func;
#else
			__ftrace_trace_function = func;
#endif
		}
231
	}
232

Steven Rostedt's avatar
Steven Rostedt committed
233
	return 0;
234
235
}

236
237
238
239
240
static void ftrace_update_pid_func(void)
{
	ftrace_func_t func;

	if (ftrace_trace_function == ftrace_stub)
241
		return;
242

243
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
244
	func = ftrace_trace_function;
245
246
247
#else
	func = __ftrace_trace_function;
#endif
248

249
	if (!list_empty(&ftrace_pids)) {
250
251
252
		set_ftrace_pid_function(func);
		func = ftrace_pid_func;
	} else {
253
254
		if (func == ftrace_pid_func)
			func = ftrace_pid_function;
255
256
257
258
259
260
261
262
263
	}

#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
	ftrace_trace_function = func;
#else
	__ftrace_trace_function = func;
#endif
}

264
265
266
267
268
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
269
270
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
271
	unsigned long long		time_squared;
272
#endif
273
274
};

275
276
277
278
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
279
280
};

281
282
283
284
285
286
287
288
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

289
290
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
291

292
293
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
294

295
296
297
298
static int ftrace_profile_bits __read_mostly;
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
299
300
static DEFINE_MUTEX(ftrace_profile_lock);

301
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
302
303
304

#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */

305
306
307
static void *
function_stat_next(void *v, int idx)
{
308
309
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
310

311
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
312
313

 again:
Li Zefan's avatar
Li Zefan committed
314
315
316
	if (idx != 0)
		rec++;

317
318
319
320
321
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
322
323
		if (!rec->counter)
			goto again;
324
325
326
327
328
329
330
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
331
332
333
334
335
336
337
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
338
339
}

340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
356
357
static int function_stat_cmp(void *p1, void *p2)
{
358
359
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
360
361
362
363
364
365
366
367

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
368
#endif
369
370
371

static int function_stat_headers(struct seq_file *m)
{
372
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
373
	seq_printf(m, "  Function                               "
374
		   "Hit    Time            Avg             s^2\n"
375
		      "  --------                               "
376
		   "---    ----            ---             ---\n");
377
#else
378
379
	seq_printf(m, "  Function                               Hit\n"
		      "  --------                               ---\n");
380
#endif
381
382
383
384
385
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
386
	struct ftrace_profile *rec = v;
387
	char str[KSYM_SYMBOL_LEN];
388
	int ret = 0;
389
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
390
391
	static struct trace_seq s;
	unsigned long long avg;
392
	unsigned long long stddev;
393
#endif
394
395
396
397
398
399
400
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
401
402

	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
403
404
405
406
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	seq_printf(m, "    ");
407
408
409
	avg = rec->time;
	do_div(avg, rec->counter);

410
411
412
413
414
415
416
417
418
419
420
421
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
		stddev = rec->time_squared - rec->counter * avg * avg;
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
		do_div(stddev, (rec->counter - 1) * 1000);
	}

422
423
424
425
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
426
427
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
428
429
430
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
431
432
out:
	mutex_unlock(&ftrace_profile_lock);
433

434
	return ret;
435
436
}

437
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
438
{
439
	struct ftrace_profile_page *pg;
440

441
	pg = stat->pages = stat->start;
442

443
444
445
446
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
447
448
	}

449
	memset(stat->hash, 0,
450
451
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
452

453
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
454
455
{
	struct ftrace_profile_page *pg;
456
457
	int functions;
	int pages;
458
	int i;
459

460
	/* If we already allocated, do nothing */
461
	if (stat->pages)
462
		return 0;
463

464
465
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
466
		return -ENOMEM;
467

468
469
470
471
472
473
474
475
476
477
478
479
480
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

481
	pg = stat->start = stat->pages;
482

483
484
485
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

	for (i = 0; i < pages; i++) {
486
487
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
488
			goto out_free;
489
490
491
492
		pg = pg->next;
	}

	return 0;
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	free_page((unsigned long)stat->pages);
	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
508
509
}

510
static int ftrace_profile_init_cpu(int cpu)
511
{
512
	struct ftrace_profile_stat *stat;
513
	int size;
514

515
516
517
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
518
		/* If the profile is already created, simply reset it */
519
		ftrace_profile_reset(stat);
520
521
		return 0;
	}
522

523
524
525
526
527
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
528

529
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
530

531
	if (!stat->hash)
532
533
		return -ENOMEM;

534
535
	if (!ftrace_profile_bits) {
		size--;
536

537
538
539
		for (; size; size >>= 1)
			ftrace_profile_bits++;
	}
540

541
	/* Preallocate the function profiling pages */
542
543
544
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
545
546
547
548
		return -ENOMEM;
	}

	return 0;
549
550
}

551
552
553
554
555
556
557
558
559
560
561
562
563
564
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

	for_each_online_cpu(cpu) {
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

565
/* interrupts must be disabled */
566
567
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
568
{
569
	struct ftrace_profile *rec;
570
571
572
573
574
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, ftrace_profile_bits);
575
	hhd = &stat->hash[key];
576
577
578
579
580
581

	if (hlist_empty(hhd))
		return NULL;

	hlist_for_each_entry_rcu(rec, n, hhd, node) {
		if (rec->ip == ip)
582
583
584
585
586
587
			return rec;
	}

	return NULL;
}

588
589
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
590
591
592
593
{
	unsigned long key;

	key = hash_long(rec->ip, ftrace_profile_bits);
594
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
595
596
}

597
598
599
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
600
static struct ftrace_profile *
601
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
602
603
604
{
	struct ftrace_profile *rec = NULL;

605
	/* prevent recursion (from NMIs) */
606
	if (atomic_inc_return(&stat->disabled) != 1)
607
608
609
		goto out;

	/*
610
611
	 * Try to find the function again since an NMI
	 * could have added it
612
	 */
613
	rec = ftrace_find_profiled_func(stat, ip);
614
	if (rec)
615
		goto out;
616

617
618
619
620
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
621
	}
622

623
	rec = &stat->pages->records[stat->pages->index++];
624
	rec->ip = ip;
625
	ftrace_add_profile(stat, rec);
626

627
 out:
628
	atomic_dec(&stat->disabled);
629
630
631
632
633
634
635

	return rec;
}

static void
function_profile_call(unsigned long ip, unsigned long parent_ip)
{
636
	struct ftrace_profile_stat *stat;
637
	struct ftrace_profile *rec;
638
639
640
641
642
643
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
644
645

	stat = &__get_cpu_var(ftrace_profile_stats);
646
	if (!stat->hash || !ftrace_profile_enabled)
647
648
649
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
650
	if (!rec) {
651
		rec = ftrace_profile_alloc(stat, ip);
652
653
654
		if (!rec)
			goto out;
	}
655
656
657
658
659
660

	rec->counter++;
 out:
	local_irq_restore(flags);
}

661
662
663
664
665
666
667
668
669
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
	function_profile_call(trace->func, 0);
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
670
	struct ftrace_profile_stat *stat;
671
	unsigned long long calltime;
672
	struct ftrace_profile *rec;
673
	unsigned long flags;
674
675

	local_irq_save(flags);
676
	stat = &__get_cpu_var(ftrace_profile_stats);
677
	if (!stat->hash || !ftrace_profile_enabled)
678
679
		goto out;

680
681
682
683
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
	calltime = trace->rettime - trace->calltime;

	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

701
	rec = ftrace_find_profiled_func(stat, trace->func);
702
	if (rec) {
703
		rec->time += calltime;
704
705
		rec->time_squared += calltime * calltime;
	}
706

707
 out:
708
709
710
711
712
713
714
715
716
717
718
719
720
721
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
722
723
static struct ftrace_ops ftrace_profile_ops __read_mostly =
{
724
	.func		= function_profile_call,
725
726
};

727
728
729
730
731
732
733
734
735
736
737
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

738
739
740
741
742
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
743
	char buf[64];		/* big enough to hold a number */
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	ret = strict_strtoul(buf, 10, &val);
	if (ret < 0)
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
763
764
765
766
767
768
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

769
770
771
772
773
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
774
775
776
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
777
778
779
780
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
781
			unregister_ftrace_profiler();
782
783
		}
	}
784
 out:
785
786
	mutex_unlock(&ftrace_profile_lock);

787
	*ppos += cnt;
788
789
790
791

	return cnt;
}

792
793
794
795
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
796
	char buf[64];		/* big enough to hold a number */
797
798
799
800
801
802
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

803
804
805
806
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
807
	.llseek		= default_llseek,
808
809
};

810
811
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
812
813
814
815
816
817
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
818
819
};

820
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
821
{
822
	struct ftrace_profile_stat *stat;
823
	struct dentry *entry;
824
	char *name;
825
	int ret;
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

		/* allocate enough for function name + cpu number */
		name = kmalloc(32, GFP_KERNEL);
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		snprintf(name, 32, "function%d", cpu);
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
854
855
856
857
858
859
860
861
862
863
	}

	entry = debugfs_create_file("function_profile_enabled", 0644,
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
		pr_warning("Could not create debugfs "
			   "'function_profile_enabled' entry\n");
}

#else /* CONFIG_FUNCTION_PROFILER */
864
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
865
866
867
868
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

#ifdef CONFIG_DYNAMIC_FTRACE

#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
	struct rcu_head		rcu;
};

enum {
	FTRACE_ENABLE_CALLS		= (1 << 0),
	FTRACE_DISABLE_CALLS		= (1 << 1),
	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
892
893
	FTRACE_START_FUNC_RET		= (1 << 3),
	FTRACE_STOP_FUNC_RET		= (1 << 4),
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
};

static int ftrace_filtered;

static struct dyn_ftrace *ftrace_new_addrs;

static DEFINE_MUTEX(ftrace_regex_lock);

struct ftrace_page {
	struct ftrace_page	*next;
	int			index;
	struct dyn_ftrace	records[];
};

#define ENTRIES_PER_PAGE \
  ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

static struct dyn_ftrace *ftrace_free_records;

/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

Ingo Molnar's avatar
Ingo Molnar committed
933
static void ftrace_free_rec(struct dyn_ftrace *rec)
934
{
935
	rec->freelist = ftrace_free_records;
936
937
938
939
	ftrace_free_records = rec;
	rec->flags |= FTRACE_FL_FREE;
}

Ingo Molnar's avatar
Ingo Molnar committed
940
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
941
{
942
943
944
945
946
947
948
	struct dyn_ftrace *rec;

	/* First check for freed records */
	if (ftrace_free_records) {
		rec = ftrace_free_records;

		if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
949
			FTRACE_WARN_ON_ONCE(1);
950
951
952
953
			ftrace_free_records = NULL;
			return NULL;
		}

954
		ftrace_free_records = rec->freelist;
955
956
957
958
		memset(rec, 0, sizeof(*rec));
		return rec;
	}

959
	if (ftrace_pages->index == ENTRIES_PER_PAGE) {
Steven Rostedt's avatar
Steven Rostedt committed
960
961
962
963
964
965
966
		if (!ftrace_pages->next) {
			/* allocate another page */
			ftrace_pages->next =
				(void *)get_zeroed_page(GFP_KERNEL);
			if (!ftrace_pages->next)
				return NULL;
		}
967
968
969
970
971
972
		ftrace_pages = ftrace_pages->next;
	}

	return &ftrace_pages->records[ftrace_pages->index++];
}

Steven Rostedt's avatar
Steven Rostedt committed
973
static struct dyn_ftrace *
974
ftrace_record_ip(unsigned long ip)
975
{
Steven Rostedt's avatar
Steven Rostedt committed
976
	struct dyn_ftrace *rec;
977

978
	if (ftrace_disabled)
Steven Rostedt's avatar
Steven Rostedt committed
979
		return NULL;
980

Steven Rostedt's avatar
Steven Rostedt committed
981
982
983
	rec = ftrace_alloc_dyn_node(ip);
	if (!rec)
		return NULL;
984

Steven Rostedt's avatar
Steven Rostedt committed
985
	rec->ip = ip;
986
	rec->newlist = ftrace_new_addrs;
987
	ftrace_new_addrs = rec;
988

Steven Rostedt's avatar
Steven Rostedt committed
989
	return rec;
990
991
}

992
993
994
995
996
997
998
999
1000
1001
static void print_ip_ins(const char *fmt, unsigned char *p)
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1002
static void ftrace_bug(int failed, unsigned long ip)
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
{
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		printk(KERN_CONT "\n");
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
}

1029

1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
/* Return 1 if the address range is reserved for ftrace */
int ftrace_text_reserved(void *start, void *end)
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	do_for_each_ftrace_rec(pg, rec) {
		if (rec->ip <= (unsigned long)end &&
		    rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
			return 1;
	} while_for_each_ftrace_rec();
	return 0;
}


1045
static int
1046
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1047
{
1048
	unsigned long ftrace_addr;
1049
	unsigned long flag = 0UL;
1050

1051
	ftrace_addr = (unsigned long)FTRACE_ADDR;
1052

Steven Rostedt's avatar
Steven Rostedt committed
1053
	/*
1054
1055
	 * If this record is not to be traced or we want to disable it,
	 * then disable it.
Steven Rostedt's avatar
Steven Rostedt committed
1056
	 *
1057
	 * If we want to enable it and filtering is off, then enable it.
Steven Rostedt's avatar
Steven Rostedt committed
1058
	 *
1059
1060
	 * If we want to enable it and filtering is on, enable it only if
	 * it's filtered
Steven Rostedt's avatar
Steven Rostedt committed
1061
	 */
1062
1063
1064
1065
	if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
		if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
			flag = FTRACE_FL_ENABLED;
	}
Steven Rostedt's avatar
Steven Rostedt committed
1066

1067
1068
1069
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
		return 0;
Steven Rostedt's avatar
Steven Rostedt committed
1070

1071
1072
1073
	if (flag) {
		rec->flags |= FTRACE_FL_ENABLED;
		return ftrace_make_call(rec, ftrace_addr);
1074
1075
	}

1076
1077
	rec->flags &= ~FTRACE_FL_ENABLED;
	return ftrace_make_nop(NULL, rec, ftrace_addr);
1078
1079
}

Ingo Molnar's avatar
Ingo Molnar committed
1080
static void ftrace_replace_code(int enable)
1081
1082
1083
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
Steven Rostedt's avatar
Steven Rostedt committed
1084
	int failed;
1085

1086
1087
1088
	if (unlikely(ftrace_disabled))
		return;

1089
	do_for_each_ftrace_rec(pg, rec) {
1090
1091
		/* Skip over free records */
		if (rec->flags & FTRACE_FL_FREE)
1092
1093
1094
			continue;

		failed = __ftrace_replace_code(rec, enable);
1095
		if (failed) {
1096
1097
1098
			ftrace_bug(failed, rec->ip);
			/* Stop processing */
			return;
1099
		}
1100
	} while_for_each_ftrace_rec();
1101
1102
}

1103
static int
1104
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1105
1106
{
	unsigned long ip;
1107
	int ret;
1108
1109
1110

	ip = rec->ip;

1111
1112
1113
	if (unlikely(ftrace_disabled))
		return 0;

1114
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1115
	if (ret) {
1116
		ftrace_bug(ret, ip);
1117
		return 0;
1118
	}
1119
	return 1;
1120
1121
}

1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

Ingo Molnar's avatar
Ingo Molnar committed
1140
static int __ftrace_modify_code(void *data)
1141
{
1142
1143
	int *command = data;

1144
	if (*command & FTRACE_ENABLE_CALLS)
1145
		ftrace_replace_code(1);
1146
	else if (*command & FTRACE_DISABLE_CALLS)
1147
1148
1149
1150