cpufreq.c 65.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 *  linux/drivers/cpufreq/cpufreq.c
 *
 *  Copyright (C) 2001 Russell King
 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6
 *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds's avatar
Linus Torvalds committed
7
 *
8
 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9
 *	Added handling for CPU hotplug
10
11
 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
 *	Fix handling for CPU hotplug -- affected CPUs
12
 *
Linus Torvalds's avatar
Linus Torvalds committed
13
14
15
16
17
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

Viresh Kumar's avatar
Viresh Kumar committed
18
19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/cpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
23
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/device.h>
24
25
26
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
27
#include <linux/mutex.h>
28
#include <linux/slab.h>
29
#include <linux/suspend.h>
30
#include <linux/syscore_ops.h>
31
#include <linux/tick.h>
32
33
#include <trace/events/power.h>

34
static LIST_HEAD(cpufreq_policy_list);
35
36
37
38
39
40
41

static inline bool policy_is_inactive(struct cpufreq_policy *policy)
{
	return cpumask_empty(policy->cpus);
}

/* Macros to iterate over CPU policies */
42
43
44
#define for_each_suitable_policy(__policy, __active)			 \
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
		if ((__active) == !policy_is_inactive(__policy))
45
46
47
48
49
50
51

#define for_each_active_policy(__policy)		\
	for_each_suitable_policy(__policy, true)
#define for_each_inactive_policy(__policy)		\
	for_each_suitable_policy(__policy, false)

#define for_each_policy(__policy)			\
52
53
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)

54
55
56
57
58
/* Iterate over governors */
static LIST_HEAD(cpufreq_governor_list);
#define for_each_governor(__governor)				\
	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)

Linus Torvalds's avatar
Linus Torvalds committed
59
/**
Dave Jones's avatar
Dave Jones committed
60
 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds's avatar
Linus Torvalds committed
61
62
63
 * level driver of CPUFreq support, and its spinlock. This lock
 * also protects the cpufreq_cpu_data array.
 */
64
static struct cpufreq_driver *cpufreq_driver;
65
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
66
67
static DEFINE_RWLOCK(cpufreq_driver_lock);

68
69
/* Flag to suspend/resume CPUFreq governors */
static bool cpufreq_suspended;
Linus Torvalds's avatar
Linus Torvalds committed
70

71
72
73
74
75
static inline bool has_target(void)
{
	return cpufreq_driver->target_index || cpufreq_driver->target;
}

Linus Torvalds's avatar
Linus Torvalds committed
76
/* internal prototypes */
77
static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
78
static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
79
static int cpufreq_start_governor(struct cpufreq_policy *policy);
Linus Torvalds's avatar
Linus Torvalds committed
80

81
static inline void cpufreq_exit_governor(struct cpufreq_policy *policy)
82
{
83
	(void)cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
84
}
Linus Torvalds's avatar
Linus Torvalds committed
85

86
87
88
89
90
static inline void cpufreq_stop_governor(struct cpufreq_policy *policy)
{
	(void)cpufreq_governor(policy, CPUFREQ_GOV_STOP);
}

Linus Torvalds's avatar
Linus Torvalds committed
91
/**
92
93
 * Two notifier lists: the "policy" list is involved in the
 * validation process for a new CPU frequency policy; the
Linus Torvalds's avatar
Linus Torvalds committed
94
95
96
97
 * "transition" list for kernel code that needs to handle
 * changes to devices when the CPU clock speed changes.
 * The mutex locks both lists.
 */
98
static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
99
static struct srcu_notifier_head cpufreq_transition_notifier_list;
Linus Torvalds's avatar
Linus Torvalds committed
100

101
static bool init_cpufreq_transition_notifier_list_called;
102
103
104
static int __init init_cpufreq_transition_notifier_list(void)
{
	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
105
	init_cpufreq_transition_notifier_list_called = true;
106
107
	return 0;
}
108
pure_initcall(init_cpufreq_transition_notifier_list);
Linus Torvalds's avatar
Linus Torvalds committed
109

110
static int off __read_mostly;
111
static int cpufreq_disabled(void)
112
113
114
115
116
117
118
{
	return off;
}
void disable_cpufreq(void)
{
	off = 1;
}
119
static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
120

121
122
bool have_governor_per_policy(void)
{
123
	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
124
}
125
EXPORT_SYMBOL_GPL(have_governor_per_policy);
126

127
128
129
130
131
132
133
134
135
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
{
	if (have_governor_per_policy())
		return &policy->kobj;
	else
		return cpufreq_global_kobject;
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);

136
137
138
139
140
141
142
143
144
struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

	return policy && !policy_is_inactive(policy) ?
		policy->freq_table : NULL;
}
EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);

145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
{
	u64 idle_time;
	u64 cur_wall_time;
	u64 busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());

	busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];

	idle_time = cur_wall_time - busy_time;
	if (wall)
		*wall = cputime_to_usecs(cur_wall_time);

	return cputime_to_usecs(idle_time);
}

u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);
	else if (!io_busy)
		idle_time += get_cpu_iowait_time_us(cpu, wall);

	return idle_time;
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);

180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
/*
 * This is a generic cpufreq init() routine which can be used by cpufreq
 * drivers of SMP systems. It will do following:
 * - validate & show freq table passed
 * - set policies transition latency
 * - policy->cpus with all possible CPUs
 */
int cpufreq_generic_init(struct cpufreq_policy *policy,
		struct cpufreq_frequency_table *table,
		unsigned int transition_latency)
{
	int ret;

	ret = cpufreq_table_validate_and_show(policy, table);
	if (ret) {
		pr_err("%s: invalid frequency table: %d\n", __func__, ret);
		return ret;
	}

	policy->cpuinfo.transition_latency = transition_latency;

	/*
202
	 * The driver only supports the SMP configuration where all processors
203
204
205
206
207
208
209
210
	 * share the clock and voltage and clock.
	 */
	cpumask_setall(policy->cpus);

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);

211
struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
212
213
214
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

215
216
	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
}
217
EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
218
219
220
221
222

unsigned int cpufreq_generic_get(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);

223
	if (!policy || IS_ERR(policy->clk)) {
224
225
		pr_err("%s: No %s associated to cpu: %d\n",
		       __func__, policy ? "clk" : "policy", cpu);
226
227
228
229
230
231
232
		return 0;
	}

	return clk_get_rate(policy->clk) / 1000;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_get);

233
234
235
236
237
238
239
240
241
242
243
244
245
/**
 * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 *
 * @cpu: cpu to find policy for.
 *
 * This returns policy for 'cpu', returns NULL if it doesn't exist.
 * It also increments the kobject reference count to mark it busy and so would
 * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 * freed as that depends on the kobj count.
 *
 * Return: A valid policy on success, otherwise NULL on failure.
 */
246
struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
247
{
248
	struct cpufreq_policy *policy = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
249
250
	unsigned long flags;

251
	if (WARN_ON(cpu >= nr_cpu_ids))
252
253
		return NULL;

Linus Torvalds's avatar
Linus Torvalds committed
254
	/* get the cpufreq driver */
255
	read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
256

257
258
	if (cpufreq_driver) {
		/* get the CPU */
259
		policy = cpufreq_cpu_get_raw(cpu);
260
261
262
		if (policy)
			kobject_get(&policy->kobj);
	}
Linus Torvalds's avatar
Linus Torvalds committed
263

264
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
265

266
	return policy;
267
}
Linus Torvalds's avatar
Linus Torvalds committed
268
269
EXPORT_SYMBOL_GPL(cpufreq_cpu_get);

270
271
272
273
274
275
276
277
/**
 * cpufreq_cpu_put: Decrements the usage count of a policy
 *
 * @policy: policy earlier returned by cpufreq_cpu_get().
 *
 * This decrements the kobject reference count incremented earlier by calling
 * cpufreq_cpu_get().
 */
278
void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
279
{
280
	kobject_put(&policy->kobj);
Linus Torvalds's avatar
Linus Torvalds committed
281
282
283
284
285
286
287
288
289
290
291
292
}
EXPORT_SYMBOL_GPL(cpufreq_cpu_put);

/*********************************************************************
 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 *********************************************************************/

/**
 * adjust_jiffies - adjust the system "loops_per_jiffy"
 *
 * This function alters the system "loops_per_jiffy" for the clock
 * speed change. Note that loops_per_jiffy cannot be updated on SMP
293
 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds's avatar
Linus Torvalds committed
294
295
 * per-CPU loops_per_jiffy value wherever possible.
 */
296
static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds's avatar
Linus Torvalds committed
297
{
298
299
300
301
#ifndef CONFIG_SMP
	static unsigned long l_p_j_ref;
	static unsigned int l_p_j_ref_freq;

Linus Torvalds's avatar
Linus Torvalds committed
302
303
304
305
306
307
	if (ci->flags & CPUFREQ_CONST_LOOPS)
		return;

	if (!l_p_j_ref_freq) {
		l_p_j_ref = loops_per_jiffy;
		l_p_j_ref_freq = ci->old;
308
309
		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
			 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds's avatar
Linus Torvalds committed
310
	}
311
	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
312
313
		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
								ci->new);
314
315
		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
			 loops_per_jiffy, ci->new);
Linus Torvalds's avatar
Linus Torvalds committed
316
317
	}
#endif
318
}
Linus Torvalds's avatar
Linus Torvalds committed
319

320
static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
321
		struct cpufreq_freqs *freqs, unsigned int state)
Linus Torvalds's avatar
Linus Torvalds committed
322
323
324
{
	BUG_ON(irqs_disabled());

325
326
327
	if (cpufreq_disabled())
		return;

328
	freqs->flags = cpufreq_driver->flags;
329
	pr_debug("notification %u of frequency transition to %u kHz\n",
330
		 state, freqs->new);
Linus Torvalds's avatar
Linus Torvalds committed
331
332

	switch (state) {
333

Linus Torvalds's avatar
Linus Torvalds committed
334
	case CPUFREQ_PRECHANGE:
335
		/* detect if the driver reported a value as "old frequency"
336
337
		 * which is not equal to what the cpufreq core thinks is
		 * "old frequency".
Linus Torvalds's avatar
Linus Torvalds committed
338
		 */
339
		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
340
341
			if ((policy) && (policy->cpu == freqs->cpu) &&
			    (policy->cur) && (policy->cur != freqs->old)) {
342
343
				pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
					 freqs->old, policy->cur);
344
				freqs->old = policy->cur;
Linus Torvalds's avatar
Linus Torvalds committed
345
346
			}
		}
347
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
348
				CPUFREQ_PRECHANGE, freqs);
Linus Torvalds's avatar
Linus Torvalds committed
349
350
		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
		break;
351

Linus Torvalds's avatar
Linus Torvalds committed
352
353
	case CPUFREQ_POSTCHANGE:
		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
354
355
		pr_debug("FREQ: %lu - CPU: %lu\n",
			 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
356
		trace_cpu_frequency(freqs->new, freqs->cpu);
357
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
358
				CPUFREQ_POSTCHANGE, freqs);
359
360
		if (likely(policy) && likely(policy->cpu == freqs->cpu))
			policy->cur = freqs->new;
Linus Torvalds's avatar
Linus Torvalds committed
361
362
363
		break;
	}
}
364

365
366
367
368
369
370
371
372
/**
 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 * on frequency transition.
 *
 * This function calls the transition notifiers and the "adjust_jiffies"
 * function. It is called twice on all CPU frequency changes that have
 * external effects.
 */
373
static void cpufreq_notify_transition(struct cpufreq_policy *policy,
374
375
376
377
378
		struct cpufreq_freqs *freqs, unsigned int state)
{
	for_each_cpu(freqs->cpu, policy->cpus)
		__cpufreq_notify_transition(policy, freqs, state);
}
Linus Torvalds's avatar
Linus Torvalds committed
379

380
/* Do post notifications when there are chances that transition has failed */
381
static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
382
383
384
385
386
387
388
389
390
391
392
		struct cpufreq_freqs *freqs, int transition_failed)
{
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
	if (!transition_failed)
		return;

	swap(freqs->old, freqs->new);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
}

393
394
395
void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs)
{
396
397
398
399
400
401
402
403
404
405
406
407

	/*
	 * Catch double invocations of _begin() which lead to self-deadlock.
	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
	 * doesn't invoke _begin() on their behalf, and hence the chances of
	 * double invocations are very low. Moreover, there are scenarios
	 * where these checks can emit false-positive warnings in these
	 * drivers; so we avoid that by skipping them altogether.
	 */
	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
				&& current == policy->transition_task);

408
409
410
411
412
413
414
415
416
417
418
wait:
	wait_event(policy->transition_wait, !policy->transition_ongoing);

	spin_lock(&policy->transition_lock);

	if (unlikely(policy->transition_ongoing)) {
		spin_unlock(&policy->transition_lock);
		goto wait;
	}

	policy->transition_ongoing = true;
419
	policy->transition_task = current;
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435

	spin_unlock(&policy->transition_lock);

	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);

void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs, int transition_failed)
{
	if (unlikely(WARN_ON(!policy->transition_ongoing)))
		return;

	cpufreq_notify_post_transition(policy, freqs, transition_failed);

	policy->transition_ongoing = false;
436
	policy->transition_task = NULL;
437
438
439
440
441

	wake_up(&policy->transition_wait);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);

442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
/*
 * Fast frequency switching status count.  Positive means "enabled", negative
 * means "disabled" and 0 means "not decided yet".
 */
static int cpufreq_fast_switch_count;
static DEFINE_MUTEX(cpufreq_fast_switch_lock);

static void cpufreq_list_transition_notifiers(void)
{
	struct notifier_block *nb;

	pr_info("Registered transition notifiers:\n");

	mutex_lock(&cpufreq_transition_notifier_list.mutex);

	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
		pr_info("%pF\n", nb->notifier_call);

	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
}

/**
 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
 * @policy: cpufreq policy to enable fast frequency switching for.
 *
 * Try to enable fast frequency switching for @policy.
 *
 * The attempt will fail if there is at least one transition notifier registered
 * at this point, as fast frequency switching is quite fundamentally at odds
 * with transition notifiers.  Thus if successful, it will make registration of
 * transition notifiers fail going forward.
 */
void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
{
	lockdep_assert_held(&policy->rwsem);

	if (!policy->fast_switch_possible)
		return;

	mutex_lock(&cpufreq_fast_switch_lock);
	if (cpufreq_fast_switch_count >= 0) {
		cpufreq_fast_switch_count++;
		policy->fast_switch_enabled = true;
	} else {
		pr_warn("CPU%u: Fast frequency switching not enabled\n",
			policy->cpu);
		cpufreq_list_transition_notifiers();
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);

494
495
496
497
498
/**
 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
 * @policy: cpufreq policy to disable fast frequency switching for.
 */
void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
499
500
501
502
503
504
505
506
507
{
	mutex_lock(&cpufreq_fast_switch_lock);
	if (policy->fast_switch_enabled) {
		policy->fast_switch_enabled = false;
		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
			cpufreq_fast_switch_count--;
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
508
EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
Linus Torvalds's avatar
Linus Torvalds committed
509
510
511
512

/*********************************************************************
 *                          SYSFS INTERFACE                          *
 *********************************************************************/
513
static ssize_t show_boost(struct kobject *kobj,
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
				 struct attribute *attr, char *buf)
{
	return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
}

static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
				  const char *buf, size_t count)
{
	int ret, enable;

	ret = sscanf(buf, "%d", &enable);
	if (ret != 1 || enable < 0 || enable > 1)
		return -EINVAL;

	if (cpufreq_boost_trigger_state(enable)) {
529
530
		pr_err("%s: Cannot %s BOOST!\n",
		       __func__, enable ? "enable" : "disable");
531
532
533
		return -EINVAL;
	}

534
535
	pr_debug("%s: cpufreq BOOST %s\n",
		 __func__, enable ? "enabled" : "disabled");
536
537
538
539

	return count;
}
define_one_global_rw(boost);
Linus Torvalds's avatar
Linus Torvalds committed
540

541
static struct cpufreq_governor *find_governor(const char *str_governor)
542
543
544
{
	struct cpufreq_governor *t;

545
	for_each_governor(t)
546
		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
547
548
549
550
551
			return t;

	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
552
553
554
/**
 * cpufreq_parse_governor - parse a governor string
 */
Dave Jones's avatar
Dave Jones committed
555
static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
Linus Torvalds's avatar
Linus Torvalds committed
556
557
				struct cpufreq_governor **governor)
{
558
	int err = -EINVAL;
559
560

	if (cpufreq_driver->setpolicy) {
561
		if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
562
			*policy = CPUFREQ_POLICY_PERFORMANCE;
563
			err = 0;
564
		} else if (!strncasecmp(str_governor, "powersave",
565
						CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
566
			*policy = CPUFREQ_POLICY_POWERSAVE;
567
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
568
		}
569
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
570
		struct cpufreq_governor *t;
571

572
		mutex_lock(&cpufreq_governor_mutex);
573

574
		t = find_governor(str_governor);
575

576
		if (t == NULL) {
577
			int ret;
578

579
580
581
			mutex_unlock(&cpufreq_governor_mutex);
			ret = request_module("cpufreq_%s", str_governor);
			mutex_lock(&cpufreq_governor_mutex);
582

583
			if (ret == 0)
584
				t = find_governor(str_governor);
585
586
		}

587
588
589
		if (t != NULL) {
			*governor = t;
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
590
		}
591

592
		mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
593
	}
594
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
595
596
597
}

/**
598
599
 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 * print out cpufreq information
Linus Torvalds's avatar
Linus Torvalds committed
600
601
602
603
604
 *
 * Write out information from cpufreq_driver->policy[cpu]; object must be
 * "unsigned int".
 */

605
606
#define show_one(file_name, object)			\
static ssize_t show_##file_name				\
Dave Jones's avatar
Dave Jones committed
607
(struct cpufreq_policy *policy, char *buf)		\
608
{							\
609
	return sprintf(buf, "%u\n", policy->object);	\
Linus Torvalds's avatar
Linus Torvalds committed
610
611
612
613
}

show_one(cpuinfo_min_freq, cpuinfo.min_freq);
show_one(cpuinfo_max_freq, cpuinfo.max_freq);
614
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds's avatar
Linus Torvalds committed
615
616
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
617

618
static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
619
620
621
622
623
624
625
626
627
{
	ssize_t ret;

	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
	else
		ret = sprintf(buf, "%u\n", policy->cur);
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
628

629
static int cpufreq_set_policy(struct cpufreq_policy *policy,
630
				struct cpufreq_policy *new_policy);
631

Linus Torvalds's avatar
Linus Torvalds committed
632
633
634
635
636
/**
 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 */
#define store_one(file_name, object)			\
static ssize_t store_##file_name					\
Dave Jones's avatar
Dave Jones committed
637
(struct cpufreq_policy *policy, const char *buf, size_t count)		\
Linus Torvalds's avatar
Linus Torvalds committed
638
{									\
639
	int ret, temp;							\
Linus Torvalds's avatar
Linus Torvalds committed
640
641
	struct cpufreq_policy new_policy;				\
									\
642
	memcpy(&new_policy, policy, sizeof(*policy));			\
Linus Torvalds's avatar
Linus Torvalds committed
643
									\
644
	ret = sscanf(buf, "%u", &new_policy.object);			\
Linus Torvalds's avatar
Linus Torvalds committed
645
646
647
	if (ret != 1)							\
		return -EINVAL;						\
									\
648
	temp = new_policy.object;					\
649
	ret = cpufreq_set_policy(policy, &new_policy);		\
650
651
	if (!ret)							\
		policy->user_policy.object = temp;			\
Linus Torvalds's avatar
Linus Torvalds committed
652
653
654
655
									\
	return ret ? ret : count;					\
}

656
657
store_one(scaling_min_freq, min);
store_one(scaling_max_freq, max);
Linus Torvalds's avatar
Linus Torvalds committed
658
659
660
661

/**
 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 */
Dave Jones's avatar
Dave Jones committed
662
663
static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
					char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
664
{
665
	unsigned int cur_freq = __cpufreq_get(policy);
Linus Torvalds's avatar
Linus Torvalds committed
666
667
668
669
670
671
672
673
	if (!cur_freq)
		return sprintf(buf, "<unknown>");
	return sprintf(buf, "%u\n", cur_freq);
}

/**
 * show_scaling_governor - show the current policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
674
static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
675
{
676
	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds's avatar
Linus Torvalds committed
677
678
679
680
		return sprintf(buf, "powersave\n");
	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
		return sprintf(buf, "performance\n");
	else if (policy->governor)
681
		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
682
				policy->governor->name);
Linus Torvalds's avatar
Linus Torvalds committed
683
684
685
686
687
688
	return -EINVAL;
}

/**
 * store_scaling_governor - store policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
689
690
static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
					const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
691
{
692
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
693
694
695
	char	str_governor[16];
	struct cpufreq_policy new_policy;

696
	memcpy(&new_policy, policy, sizeof(*policy));
Linus Torvalds's avatar
Linus Torvalds committed
697

698
	ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds's avatar
Linus Torvalds committed
699
700
701
	if (ret != 1)
		return -EINVAL;

702
703
	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
						&new_policy.governor))
Linus Torvalds's avatar
Linus Torvalds committed
704
705
		return -EINVAL;

706
	ret = cpufreq_set_policy(policy, &new_policy);
707
	return ret ? ret : count;
Linus Torvalds's avatar
Linus Torvalds committed
708
709
710
711
712
}

/**
 * show_scaling_driver - show the cpufreq driver currently loaded
 */
Dave Jones's avatar
Dave Jones committed
713
static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
714
{
715
	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds's avatar
Linus Torvalds committed
716
717
718
719
720
}

/**
 * show_scaling_available_governors - show the available CPUfreq governors
 */
Dave Jones's avatar
Dave Jones committed
721
722
static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
						char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
723
724
725
726
{
	ssize_t i = 0;
	struct cpufreq_governor *t;

727
	if (!has_target()) {
Linus Torvalds's avatar
Linus Torvalds committed
728
729
730
731
		i += sprintf(buf, "performance powersave");
		goto out;
	}

732
	for_each_governor(t) {
733
734
		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
		    - (CPUFREQ_NAME_LEN + 2)))
Linus Torvalds's avatar
Linus Torvalds committed
735
			goto out;
736
		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds's avatar
Linus Torvalds committed
737
	}
738
out:
Linus Torvalds's avatar
Linus Torvalds committed
739
740
741
	i += sprintf(&buf[i], "\n");
	return i;
}
742

743
ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
744
745
746
747
{
	ssize_t i = 0;
	unsigned int cpu;

748
	for_each_cpu(cpu, mask) {
Linus Torvalds's avatar
Linus Torvalds committed
749
750
751
752
		if (i)
			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
		if (i >= (PAGE_SIZE - 5))
753
			break;
Linus Torvalds's avatar
Linus Torvalds committed
754
755
756
757
	}
	i += sprintf(&buf[i], "\n");
	return i;
}
758
EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds's avatar
Linus Torvalds committed
759

760
761
762
763
764
765
/**
 * show_related_cpus - show the CPUs affected by each transition even if
 * hw coordination is in use
 */
static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
{
766
	return cpufreq_show_cpus(policy->related_cpus, buf);
767
768
769
770
771
772
773
}

/**
 * show_affected_cpus - show the CPUs affected by each transition
 */
static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
{
774
	return cpufreq_show_cpus(policy->cpus, buf);
775
776
}

777
static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones's avatar
Dave Jones committed
778
					const char *buf, size_t count)
779
780
781
782
{
	unsigned int freq = 0;
	unsigned int ret;

783
	if (!policy->governor || !policy->governor->store_setspeed)
784
785
786
787
788
789
790
791
792
793
794
795
796
		return -EINVAL;

	ret = sscanf(buf, "%u", &freq);
	if (ret != 1)
		return -EINVAL;

	policy->governor->store_setspeed(policy, freq);

	return count;
}

static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
{
797
	if (!policy->governor || !policy->governor->show_setspeed)
798
799
800
801
		return sprintf(buf, "<unsupported>\n");

	return policy->governor->show_setspeed(policy, buf);
}
Linus Torvalds's avatar
Linus Torvalds committed
802

803
/**
804
 * show_bios_limit - show the current cpufreq HW/BIOS limitation
805
806
807
808
809
 */
static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
{
	unsigned int limit;
	int ret;
810
811
	if (cpufreq_driver->bios_limit) {
		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
812
813
814
815
816
817
		if (!ret)
			return sprintf(buf, "%u\n", limit);
	}
	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
}

818
819
820
821
822
823
824
825
826
827
828
829
830
831
cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
cpufreq_freq_attr_ro(cpuinfo_min_freq);
cpufreq_freq_attr_ro(cpuinfo_max_freq);
cpufreq_freq_attr_ro(cpuinfo_transition_latency);
cpufreq_freq_attr_ro(scaling_available_governors);
cpufreq_freq_attr_ro(scaling_driver);
cpufreq_freq_attr_ro(scaling_cur_freq);
cpufreq_freq_attr_ro(bios_limit);
cpufreq_freq_attr_ro(related_cpus);
cpufreq_freq_attr_ro(affected_cpus);
cpufreq_freq_attr_rw(scaling_min_freq);
cpufreq_freq_attr_rw(scaling_max_freq);
cpufreq_freq_attr_rw(scaling_governor);
cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds's avatar
Linus Torvalds committed
832

Dave Jones's avatar
Dave Jones committed
833
static struct attribute *default_attrs[] = {
Linus Torvalds's avatar
Linus Torvalds committed
834
835
	&cpuinfo_min_freq.attr,
	&cpuinfo_max_freq.attr,
836
	&cpuinfo_transition_latency.attr,
Linus Torvalds's avatar
Linus Torvalds committed
837
838
839
	&scaling_min_freq.attr,
	&scaling_max_freq.attr,
	&affected_cpus.attr,
840
	&related_cpus.attr,
Linus Torvalds's avatar
Linus Torvalds committed
841
842
843
	&scaling_governor.attr,
	&scaling_driver.attr,
	&scaling_available_governors.attr,
844
	&scaling_setspeed.attr,
Linus Torvalds's avatar
Linus Torvalds committed
845
846
847
	NULL
};

848
849
#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds's avatar
Linus Torvalds committed
850

851
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
852
{
Dave Jones's avatar
Dave Jones committed
853
854
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
855
	ssize_t ret;
856

857
	down_read(&policy->rwsem);
858
	ret = fattr->show(policy, buf);
859
	up_read(&policy->rwsem);
860

Linus Torvalds's avatar
Linus Torvalds committed
861
862
863
	return ret;
}

Dave Jones's avatar
Dave Jones committed
864
865
static ssize_t store(struct kobject *kobj, struct attribute *attr,
		     const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
866
{
Dave Jones's avatar
Dave Jones committed
867
868
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
869
	ssize_t ret = -EINVAL;
870

871
872
	get_online_cpus();

873
874
	if (cpu_online(policy->cpu)) {
		down_write(&policy->rwsem);
875
		ret = fattr->store(policy, buf, count);
876
877
		up_write(&policy->rwsem);
	}
878

879
880
	put_online_cpus();

Linus Torvalds's avatar
Linus Torvalds committed
881
882
883
	return ret;
}

Dave Jones's avatar
Dave Jones committed
884
static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds's avatar
Linus Torvalds committed
885
{
Dave Jones's avatar
Dave Jones committed
886
	struct cpufreq_policy *policy = to_policy(kobj);
887
	pr_debug("last reference is dropped\n");
Linus Torvalds's avatar
Linus Torvalds committed
888
889
890
	complete(&policy->kobj_unregister);
}

891
static const struct sysfs_ops sysfs_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
892
893
894
895
896
897
898
899
900
901
	.show	= show,
	.store	= store,
};

static struct kobj_type ktype_cpufreq = {
	.sysfs_ops	= &sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= cpufreq_sysfs_release,
};

902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);

	if (!policy)
		return 0;

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return 0;

	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
}

static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return;

	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
}

/* Add/remove symlinks for all related CPUs */
932
static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
933
934
935
936
{
	unsigned int j;
	int ret = 0;

937
	/* Some related CPUs might not be present (physically hotplugged) */
938
	for_each_cpu(j, policy->real_cpus) {
939
		ret = add_cpu_dev_symlink(policy, j);
940
941
		if (ret)
			break;
942
	}
943

944
945
946
	return ret;
}

947
948
949
950
951
static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
{
	unsigned int j;

	/* Some related CPUs might not be present (physically hotplugged) */
952
	for_each_cpu(j, policy->real_cpus)
953
954
955
		remove_cpu_dev_symlink(policy, j);
}

956
static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
957
958
959
960
961
{
	struct freq_attr **drv_attr;
	int ret = 0;

	/* set up files for this cpu device */
962
	drv_attr = cpufreq_driver->attr;
963
	while (drv_attr && *drv_attr) {
964
965
		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
		if (ret)
966
			return ret;
967
968
		drv_attr++;
	}
969
	if (cpufreq_driver->get) {
970
971
		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
		if (ret)
972
			return ret;
973
	}
974
975
976

	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
	if (ret)
977
		return ret;
978

979
	if (cpufreq_driver->bios_limit) {
980
981
		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
		if (ret)
982
			return ret;
983
	}
984

985
	return cpufreq_add_dev_symlink(policy);
986
987
}

988
989
990
991
992
__weak struct cpufreq_governor *cpufreq_default_governor(void)
{
	return NULL;
}

993
static int cpufreq_init_policy(struct cpufreq_policy *policy)
994
{
995
	struct cpufreq_governor *gov = NULL;
996
997
	struct cpufreq_policy new_policy;

998
	memcpy(&new_policy, policy, sizeof(*policy));
999

1000
	/* Update governor of new_policy to the governor used before hotplug */