cpufreq.c 66.4 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 *  linux/drivers/cpufreq/cpufreq.c
 *
 *  Copyright (C) 2001 Russell King
 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6
 *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds's avatar
Linus Torvalds committed
7
 *
8
 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9
 *	Added handling for CPU hotplug
10
11
 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
 *	Fix handling for CPU hotplug -- affected CPUs
12
 *
Linus Torvalds's avatar
Linus Torvalds committed
13
14
15
16
17
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

Viresh Kumar's avatar
Viresh Kumar committed
18
19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/cpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
23
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/device.h>
24
25
26
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
27
#include <linux/mutex.h>
28
#include <linux/slab.h>
29
#include <linux/suspend.h>
30
#include <linux/syscore_ops.h>
31
#include <linux/tick.h>
32
33
#include <trace/events/power.h>

34
static LIST_HEAD(cpufreq_policy_list);
35
36
37
38
39
40
41

static inline bool policy_is_inactive(struct cpufreq_policy *policy)
{
	return cpumask_empty(policy->cpus);
}

/* Macros to iterate over CPU policies */
42
43
44
#define for_each_suitable_policy(__policy, __active)			 \
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
		if ((__active) == !policy_is_inactive(__policy))
45
46
47
48
49
50
51

#define for_each_active_policy(__policy)		\
	for_each_suitable_policy(__policy, true)
#define for_each_inactive_policy(__policy)		\
	for_each_suitable_policy(__policy, false)

#define for_each_policy(__policy)			\
52
53
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)

54
55
56
57
58
/* Iterate over governors */
static LIST_HEAD(cpufreq_governor_list);
#define for_each_governor(__governor)				\
	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)

Linus Torvalds's avatar
Linus Torvalds committed
59
/**
Dave Jones's avatar
Dave Jones committed
60
 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds's avatar
Linus Torvalds committed
61
62
63
 * level driver of CPUFreq support, and its spinlock. This lock
 * also protects the cpufreq_cpu_data array.
 */
64
static struct cpufreq_driver *cpufreq_driver;
65
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
66
67
static DEFINE_RWLOCK(cpufreq_driver_lock);

68
69
/* Flag to suspend/resume CPUFreq governors */
static bool cpufreq_suspended;
Linus Torvalds's avatar
Linus Torvalds committed
70

71
72
73
74
75
static inline bool has_target(void)
{
	return cpufreq_driver->target_index || cpufreq_driver->target;
}

Linus Torvalds's avatar
Linus Torvalds committed
76
/* internal prototypes */
77
static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
78
79
static int cpufreq_init_governor(struct cpufreq_policy *policy);
static void cpufreq_exit_governor(struct cpufreq_policy *policy);
80
static int cpufreq_start_governor(struct cpufreq_policy *policy);
81
82
static void cpufreq_stop_governor(struct cpufreq_policy *policy);
static void cpufreq_governor_limits(struct cpufreq_policy *policy);
83

Linus Torvalds's avatar
Linus Torvalds committed
84
/**
85
86
 * Two notifier lists: the "policy" list is involved in the
 * validation process for a new CPU frequency policy; the
Linus Torvalds's avatar
Linus Torvalds committed
87
88
89
90
 * "transition" list for kernel code that needs to handle
 * changes to devices when the CPU clock speed changes.
 * The mutex locks both lists.
 */
91
static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
92
static struct srcu_notifier_head cpufreq_transition_notifier_list;
Linus Torvalds's avatar
Linus Torvalds committed
93

94
static bool init_cpufreq_transition_notifier_list_called;
95
96
97
static int __init init_cpufreq_transition_notifier_list(void)
{
	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
98
	init_cpufreq_transition_notifier_list_called = true;
99
100
	return 0;
}
101
pure_initcall(init_cpufreq_transition_notifier_list);
Linus Torvalds's avatar
Linus Torvalds committed
102

103
static int off __read_mostly;
104
static int cpufreq_disabled(void)
105
106
107
108
109
110
111
{
	return off;
}
void disable_cpufreq(void)
{
	off = 1;
}
112
static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
113

114
115
bool have_governor_per_policy(void)
{
116
	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
117
}
118
EXPORT_SYMBOL_GPL(have_governor_per_policy);
119

120
121
122
123
124
125
126
127
128
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
{
	if (have_governor_per_policy())
		return &policy->kobj;
	else
		return cpufreq_global_kobject;
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);

129
130
131
132
133
134
135
136
137
struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

	return policy && !policy_is_inactive(policy) ?
		policy->freq_table : NULL;
}
EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
{
	u64 idle_time;
	u64 cur_wall_time;
	u64 busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());

	busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];

	idle_time = cur_wall_time - busy_time;
	if (wall)
		*wall = cputime_to_usecs(cur_wall_time);

	return cputime_to_usecs(idle_time);
}

u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);
	else if (!io_busy)
		idle_time += get_cpu_iowait_time_us(cpu, wall);

	return idle_time;
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
/*
 * This is a generic cpufreq init() routine which can be used by cpufreq
 * drivers of SMP systems. It will do following:
 * - validate & show freq table passed
 * - set policies transition latency
 * - policy->cpus with all possible CPUs
 */
int cpufreq_generic_init(struct cpufreq_policy *policy,
		struct cpufreq_frequency_table *table,
		unsigned int transition_latency)
{
	int ret;

	ret = cpufreq_table_validate_and_show(policy, table);
	if (ret) {
		pr_err("%s: invalid frequency table: %d\n", __func__, ret);
		return ret;
	}

	policy->cpuinfo.transition_latency = transition_latency;

	/*
195
	 * The driver only supports the SMP configuration where all processors
196
197
198
199
200
201
202
203
	 * share the clock and voltage and clock.
	 */
	cpumask_setall(policy->cpus);

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);

204
struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
205
206
207
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

208
209
	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
}
210
EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
211
212
213
214
215

unsigned int cpufreq_generic_get(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);

216
	if (!policy || IS_ERR(policy->clk)) {
217
218
		pr_err("%s: No %s associated to cpu: %d\n",
		       __func__, policy ? "clk" : "policy", cpu);
219
220
221
222
223
224
225
		return 0;
	}

	return clk_get_rate(policy->clk) / 1000;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_get);

226
227
228
229
230
231
232
233
234
235
236
237
238
/**
 * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 *
 * @cpu: cpu to find policy for.
 *
 * This returns policy for 'cpu', returns NULL if it doesn't exist.
 * It also increments the kobject reference count to mark it busy and so would
 * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 * freed as that depends on the kobj count.
 *
 * Return: A valid policy on success, otherwise NULL on failure.
 */
239
struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
240
{
241
	struct cpufreq_policy *policy = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
242
243
	unsigned long flags;

244
	if (WARN_ON(cpu >= nr_cpu_ids))
245
246
		return NULL;

Linus Torvalds's avatar
Linus Torvalds committed
247
	/* get the cpufreq driver */
248
	read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
249

250
251
	if (cpufreq_driver) {
		/* get the CPU */
252
		policy = cpufreq_cpu_get_raw(cpu);
253
254
255
		if (policy)
			kobject_get(&policy->kobj);
	}
Linus Torvalds's avatar
Linus Torvalds committed
256

257
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
258

259
	return policy;
260
}
Linus Torvalds's avatar
Linus Torvalds committed
261
262
EXPORT_SYMBOL_GPL(cpufreq_cpu_get);

263
264
265
266
267
268
269
270
/**
 * cpufreq_cpu_put: Decrements the usage count of a policy
 *
 * @policy: policy earlier returned by cpufreq_cpu_get().
 *
 * This decrements the kobject reference count incremented earlier by calling
 * cpufreq_cpu_get().
 */
271
void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
272
{
273
	kobject_put(&policy->kobj);
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278
279
280
281
282
283
284
285
}
EXPORT_SYMBOL_GPL(cpufreq_cpu_put);

/*********************************************************************
 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 *********************************************************************/

/**
 * adjust_jiffies - adjust the system "loops_per_jiffy"
 *
 * This function alters the system "loops_per_jiffy" for the clock
 * speed change. Note that loops_per_jiffy cannot be updated on SMP
286
 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds's avatar
Linus Torvalds committed
287
288
 * per-CPU loops_per_jiffy value wherever possible.
 */
289
static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds's avatar
Linus Torvalds committed
290
{
291
292
293
294
#ifndef CONFIG_SMP
	static unsigned long l_p_j_ref;
	static unsigned int l_p_j_ref_freq;

Linus Torvalds's avatar
Linus Torvalds committed
295
296
297
298
299
300
	if (ci->flags & CPUFREQ_CONST_LOOPS)
		return;

	if (!l_p_j_ref_freq) {
		l_p_j_ref = loops_per_jiffy;
		l_p_j_ref_freq = ci->old;
301
302
		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
			 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds's avatar
Linus Torvalds committed
303
	}
304
	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
305
306
		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
								ci->new);
307
308
		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
			 loops_per_jiffy, ci->new);
Linus Torvalds's avatar
Linus Torvalds committed
309
310
	}
#endif
311
}
Linus Torvalds's avatar
Linus Torvalds committed
312

313
static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
314
		struct cpufreq_freqs *freqs, unsigned int state)
Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
{
	BUG_ON(irqs_disabled());

318
319
320
	if (cpufreq_disabled())
		return;

321
	freqs->flags = cpufreq_driver->flags;
322
	pr_debug("notification %u of frequency transition to %u kHz\n",
323
		 state, freqs->new);
Linus Torvalds's avatar
Linus Torvalds committed
324
325

	switch (state) {
326

Linus Torvalds's avatar
Linus Torvalds committed
327
	case CPUFREQ_PRECHANGE:
328
		/* detect if the driver reported a value as "old frequency"
329
330
		 * which is not equal to what the cpufreq core thinks is
		 * "old frequency".
Linus Torvalds's avatar
Linus Torvalds committed
331
		 */
332
		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
333
334
			if ((policy) && (policy->cpu == freqs->cpu) &&
			    (policy->cur) && (policy->cur != freqs->old)) {
335
336
				pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
					 freqs->old, policy->cur);
337
				freqs->old = policy->cur;
Linus Torvalds's avatar
Linus Torvalds committed
338
339
			}
		}
340
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
341
				CPUFREQ_PRECHANGE, freqs);
Linus Torvalds's avatar
Linus Torvalds committed
342
343
		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
		break;
344

Linus Torvalds's avatar
Linus Torvalds committed
345
346
	case CPUFREQ_POSTCHANGE:
		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
347
348
		pr_debug("FREQ: %lu - CPU: %lu\n",
			 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
349
		trace_cpu_frequency(freqs->new, freqs->cpu);
350
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
351
				CPUFREQ_POSTCHANGE, freqs);
352
353
		if (likely(policy) && likely(policy->cpu == freqs->cpu))
			policy->cur = freqs->new;
Linus Torvalds's avatar
Linus Torvalds committed
354
355
356
		break;
	}
}
357

358
359
360
361
362
363
364
365
/**
 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 * on frequency transition.
 *
 * This function calls the transition notifiers and the "adjust_jiffies"
 * function. It is called twice on all CPU frequency changes that have
 * external effects.
 */
366
static void cpufreq_notify_transition(struct cpufreq_policy *policy,
367
368
369
370
371
		struct cpufreq_freqs *freqs, unsigned int state)
{
	for_each_cpu(freqs->cpu, policy->cpus)
		__cpufreq_notify_transition(policy, freqs, state);
}
Linus Torvalds's avatar
Linus Torvalds committed
372

373
/* Do post notifications when there are chances that transition has failed */
374
static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
375
376
377
378
379
380
381
382
383
384
385
		struct cpufreq_freqs *freqs, int transition_failed)
{
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
	if (!transition_failed)
		return;

	swap(freqs->old, freqs->new);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
}

386
387
388
void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs)
{
389
390
391
392
393
394
395
396
397
398
399
400

	/*
	 * Catch double invocations of _begin() which lead to self-deadlock.
	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
	 * doesn't invoke _begin() on their behalf, and hence the chances of
	 * double invocations are very low. Moreover, there are scenarios
	 * where these checks can emit false-positive warnings in these
	 * drivers; so we avoid that by skipping them altogether.
	 */
	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
				&& current == policy->transition_task);

401
402
403
404
405
406
407
408
409
410
411
wait:
	wait_event(policy->transition_wait, !policy->transition_ongoing);

	spin_lock(&policy->transition_lock);

	if (unlikely(policy->transition_ongoing)) {
		spin_unlock(&policy->transition_lock);
		goto wait;
	}

	policy->transition_ongoing = true;
412
	policy->transition_task = current;
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428

	spin_unlock(&policy->transition_lock);

	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);

void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs, int transition_failed)
{
	if (unlikely(WARN_ON(!policy->transition_ongoing)))
		return;

	cpufreq_notify_post_transition(policy, freqs, transition_failed);

	policy->transition_ongoing = false;
429
	policy->transition_task = NULL;
430
431
432
433
434

	wake_up(&policy->transition_wait);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);

435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
/*
 * Fast frequency switching status count.  Positive means "enabled", negative
 * means "disabled" and 0 means "not decided yet".
 */
static int cpufreq_fast_switch_count;
static DEFINE_MUTEX(cpufreq_fast_switch_lock);

static void cpufreq_list_transition_notifiers(void)
{
	struct notifier_block *nb;

	pr_info("Registered transition notifiers:\n");

	mutex_lock(&cpufreq_transition_notifier_list.mutex);

	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
		pr_info("%pF\n", nb->notifier_call);

	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
}

/**
 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
 * @policy: cpufreq policy to enable fast frequency switching for.
 *
 * Try to enable fast frequency switching for @policy.
 *
 * The attempt will fail if there is at least one transition notifier registered
 * at this point, as fast frequency switching is quite fundamentally at odds
 * with transition notifiers.  Thus if successful, it will make registration of
 * transition notifiers fail going forward.
 */
void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
{
	lockdep_assert_held(&policy->rwsem);

	if (!policy->fast_switch_possible)
		return;

	mutex_lock(&cpufreq_fast_switch_lock);
	if (cpufreq_fast_switch_count >= 0) {
		cpufreq_fast_switch_count++;
		policy->fast_switch_enabled = true;
	} else {
		pr_warn("CPU%u: Fast frequency switching not enabled\n",
			policy->cpu);
		cpufreq_list_transition_notifiers();
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);

487
488
489
490
491
/**
 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
 * @policy: cpufreq policy to disable fast frequency switching for.
 */
void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
492
493
494
495
496
497
498
499
500
{
	mutex_lock(&cpufreq_fast_switch_lock);
	if (policy->fast_switch_enabled) {
		policy->fast_switch_enabled = false;
		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
			cpufreq_fast_switch_count--;
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
501
EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
Linus Torvalds's avatar
Linus Torvalds committed
502
503
504
505

/*********************************************************************
 *                          SYSFS INTERFACE                          *
 *********************************************************************/
506
static ssize_t show_boost(struct kobject *kobj,
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
				 struct attribute *attr, char *buf)
{
	return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
}

static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
				  const char *buf, size_t count)
{
	int ret, enable;

	ret = sscanf(buf, "%d", &enable);
	if (ret != 1 || enable < 0 || enable > 1)
		return -EINVAL;

	if (cpufreq_boost_trigger_state(enable)) {
522
523
		pr_err("%s: Cannot %s BOOST!\n",
		       __func__, enable ? "enable" : "disable");
524
525
526
		return -EINVAL;
	}

527
528
	pr_debug("%s: cpufreq BOOST %s\n",
		 __func__, enable ? "enabled" : "disabled");
529
530
531
532

	return count;
}
define_one_global_rw(boost);
Linus Torvalds's avatar
Linus Torvalds committed
533

534
static struct cpufreq_governor *find_governor(const char *str_governor)
535
536
537
{
	struct cpufreq_governor *t;

538
	for_each_governor(t)
539
		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
540
541
542
543
544
			return t;

	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
545
546
547
/**
 * cpufreq_parse_governor - parse a governor string
 */
Dave Jones's avatar
Dave Jones committed
548
static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
Linus Torvalds's avatar
Linus Torvalds committed
549
550
				struct cpufreq_governor **governor)
{
551
	int err = -EINVAL;
552
553

	if (cpufreq_driver->setpolicy) {
554
		if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
555
			*policy = CPUFREQ_POLICY_PERFORMANCE;
556
			err = 0;
557
		} else if (!strncasecmp(str_governor, "powersave",
558
						CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
559
			*policy = CPUFREQ_POLICY_POWERSAVE;
560
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
561
		}
562
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
563
		struct cpufreq_governor *t;
564

565
		mutex_lock(&cpufreq_governor_mutex);
566

567
		t = find_governor(str_governor);
568

569
		if (t == NULL) {
570
			int ret;
571

572
573
574
			mutex_unlock(&cpufreq_governor_mutex);
			ret = request_module("cpufreq_%s", str_governor);
			mutex_lock(&cpufreq_governor_mutex);
575

576
			if (ret == 0)
577
				t = find_governor(str_governor);
578
579
		}

580
581
582
		if (t != NULL) {
			*governor = t;
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
583
		}
584

585
		mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
586
	}
587
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
588
589
590
}

/**
591
592
 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 * print out cpufreq information
Linus Torvalds's avatar
Linus Torvalds committed
593
594
595
596
597
 *
 * Write out information from cpufreq_driver->policy[cpu]; object must be
 * "unsigned int".
 */

598
599
#define show_one(file_name, object)			\
static ssize_t show_##file_name				\
Dave Jones's avatar
Dave Jones committed
600
(struct cpufreq_policy *policy, char *buf)		\
601
{							\
602
	return sprintf(buf, "%u\n", policy->object);	\
Linus Torvalds's avatar
Linus Torvalds committed
603
604
605
606
}

show_one(cpuinfo_min_freq, cpuinfo.min_freq);
show_one(cpuinfo_max_freq, cpuinfo.max_freq);
607
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds's avatar
Linus Torvalds committed
608
609
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
610

611
static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
612
613
614
615
616
617
618
619
620
{
	ssize_t ret;

	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
	else
		ret = sprintf(buf, "%u\n", policy->cur);
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
621

622
static int cpufreq_set_policy(struct cpufreq_policy *policy,
623
				struct cpufreq_policy *new_policy);
624

Linus Torvalds's avatar
Linus Torvalds committed
625
626
627
628
629
/**
 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 */
#define store_one(file_name, object)			\
static ssize_t store_##file_name					\
Dave Jones's avatar
Dave Jones committed
630
(struct cpufreq_policy *policy, const char *buf, size_t count)		\
Linus Torvalds's avatar
Linus Torvalds committed
631
{									\
632
	int ret, temp;							\
Linus Torvalds's avatar
Linus Torvalds committed
633
634
	struct cpufreq_policy new_policy;				\
									\
635
	memcpy(&new_policy, policy, sizeof(*policy));			\
Linus Torvalds's avatar
Linus Torvalds committed
636
									\
637
	ret = sscanf(buf, "%u", &new_policy.object);			\
Linus Torvalds's avatar
Linus Torvalds committed
638
639
640
	if (ret != 1)							\
		return -EINVAL;						\
									\
641
	temp = new_policy.object;					\
642
	ret = cpufreq_set_policy(policy, &new_policy);		\
643
644
	if (!ret)							\
		policy->user_policy.object = temp;			\
Linus Torvalds's avatar
Linus Torvalds committed
645
646
647
648
									\
	return ret ? ret : count;					\
}

649
650
store_one(scaling_min_freq, min);
store_one(scaling_max_freq, max);
Linus Torvalds's avatar
Linus Torvalds committed
651
652
653
654

/**
 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 */
Dave Jones's avatar
Dave Jones committed
655
656
static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
					char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
657
{
658
	unsigned int cur_freq = __cpufreq_get(policy);
Linus Torvalds's avatar
Linus Torvalds committed
659
660
661
662
663
664
665
666
	if (!cur_freq)
		return sprintf(buf, "<unknown>");
	return sprintf(buf, "%u\n", cur_freq);
}

/**
 * show_scaling_governor - show the current policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
667
static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
668
{
669
	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds's avatar
Linus Torvalds committed
670
671
672
673
		return sprintf(buf, "powersave\n");
	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
		return sprintf(buf, "performance\n");
	else if (policy->governor)
674
		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
675
				policy->governor->name);
Linus Torvalds's avatar
Linus Torvalds committed
676
677
678
679
680
681
	return -EINVAL;
}

/**
 * store_scaling_governor - store policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
682
683
static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
					const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
684
{
685
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
686
687
688
	char	str_governor[16];
	struct cpufreq_policy new_policy;

689
	memcpy(&new_policy, policy, sizeof(*policy));
Linus Torvalds's avatar
Linus Torvalds committed
690

691
	ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds's avatar
Linus Torvalds committed
692
693
694
	if (ret != 1)
		return -EINVAL;

695
696
	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
						&new_policy.governor))
Linus Torvalds's avatar
Linus Torvalds committed
697
698
		return -EINVAL;

699
	ret = cpufreq_set_policy(policy, &new_policy);
700
	return ret ? ret : count;
Linus Torvalds's avatar
Linus Torvalds committed
701
702
703
704
705
}

/**
 * show_scaling_driver - show the cpufreq driver currently loaded
 */
Dave Jones's avatar
Dave Jones committed
706
static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
707
{
708
	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds's avatar
Linus Torvalds committed
709
710
711
712
713
}

/**
 * show_scaling_available_governors - show the available CPUfreq governors
 */
Dave Jones's avatar
Dave Jones committed
714
715
static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
						char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
716
717
718
719
{
	ssize_t i = 0;
	struct cpufreq_governor *t;

720
	if (!has_target()) {
Linus Torvalds's avatar
Linus Torvalds committed
721
722
723
724
		i += sprintf(buf, "performance powersave");
		goto out;
	}

725
	for_each_governor(t) {
726
727
		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
		    - (CPUFREQ_NAME_LEN + 2)))
Linus Torvalds's avatar
Linus Torvalds committed
728
			goto out;
729
		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds's avatar
Linus Torvalds committed
730
	}
731
out:
Linus Torvalds's avatar
Linus Torvalds committed
732
733
734
	i += sprintf(&buf[i], "\n");
	return i;
}
735

736
ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
737
738
739
740
{
	ssize_t i = 0;
	unsigned int cpu;

741
	for_each_cpu(cpu, mask) {
Linus Torvalds's avatar
Linus Torvalds committed
742
743
744
745
		if (i)
			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
		if (i >= (PAGE_SIZE - 5))
746
			break;
Linus Torvalds's avatar
Linus Torvalds committed
747
748
749
750
	}
	i += sprintf(&buf[i], "\n");
	return i;
}
751
EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds's avatar
Linus Torvalds committed
752

753
754
755
756
757
758
/**
 * show_related_cpus - show the CPUs affected by each transition even if
 * hw coordination is in use
 */
static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
{
759
	return cpufreq_show_cpus(policy->related_cpus, buf);
760
761
762
763
764
765
766
}

/**
 * show_affected_cpus - show the CPUs affected by each transition
 */
static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
{
767
	return cpufreq_show_cpus(policy->cpus, buf);
768
769
}

770
static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones's avatar
Dave Jones committed
771
					const char *buf, size_t count)
772
773
774
775
{
	unsigned int freq = 0;
	unsigned int ret;

776
	if (!policy->governor || !policy->governor->store_setspeed)
777
778
779
780
781
782
783
784
785
786
787
788
789
		return -EINVAL;

	ret = sscanf(buf, "%u", &freq);
	if (ret != 1)
		return -EINVAL;

	policy->governor->store_setspeed(policy, freq);

	return count;
}

static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
{
790
	if (!policy->governor || !policy->governor->show_setspeed)
791
792
793
794
		return sprintf(buf, "<unsupported>\n");

	return policy->governor->show_setspeed(policy, buf);
}
Linus Torvalds's avatar
Linus Torvalds committed
795

796
/**
797
 * show_bios_limit - show the current cpufreq HW/BIOS limitation
798
799
800
801
802
 */
static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
{
	unsigned int limit;
	int ret;
803
804
	if (cpufreq_driver->bios_limit) {
		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
805
806
807
808
809
810
		if (!ret)
			return sprintf(buf, "%u\n", limit);
	}
	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
}

811
812
813
814
815
816
817
818
819
820
821
822
823
824
cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
cpufreq_freq_attr_ro(cpuinfo_min_freq);
cpufreq_freq_attr_ro(cpuinfo_max_freq);
cpufreq_freq_attr_ro(cpuinfo_transition_latency);
cpufreq_freq_attr_ro(scaling_available_governors);
cpufreq_freq_attr_ro(scaling_driver);
cpufreq_freq_attr_ro(scaling_cur_freq);
cpufreq_freq_attr_ro(bios_limit);
cpufreq_freq_attr_ro(related_cpus);
cpufreq_freq_attr_ro(affected_cpus);
cpufreq_freq_attr_rw(scaling_min_freq);
cpufreq_freq_attr_rw(scaling_max_freq);
cpufreq_freq_attr_rw(scaling_governor);
cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds's avatar
Linus Torvalds committed
825

Dave Jones's avatar
Dave Jones committed
826
static struct attribute *default_attrs[] = {
Linus Torvalds's avatar
Linus Torvalds committed
827
828
	&cpuinfo_min_freq.attr,
	&cpuinfo_max_freq.attr,
829
	&cpuinfo_transition_latency.attr,
Linus Torvalds's avatar
Linus Torvalds committed
830
831
832
	&scaling_min_freq.attr,
	&scaling_max_freq.attr,
	&affected_cpus.attr,
833
	&related_cpus.attr,
Linus Torvalds's avatar
Linus Torvalds committed
834
835
836
	&scaling_governor.attr,
	&scaling_driver.attr,
	&scaling_available_governors.attr,
837
	&scaling_setspeed.attr,
Linus Torvalds's avatar
Linus Torvalds committed
838
839
840
	NULL
};

841
842
#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds's avatar
Linus Torvalds committed
843

844
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
845
{
Dave Jones's avatar
Dave Jones committed
846
847
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
848
	ssize_t ret;
849

850
	down_read(&policy->rwsem);
851
	ret = fattr->show(policy, buf);
852
	up_read(&policy->rwsem);
853

Linus Torvalds's avatar
Linus Torvalds committed
854
855
856
	return ret;
}

Dave Jones's avatar
Dave Jones committed
857
858
static ssize_t store(struct kobject *kobj, struct attribute *attr,
		     const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
859
{
Dave Jones's avatar
Dave Jones committed
860
861
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
862
	ssize_t ret = -EINVAL;
863

864
865
	get_online_cpus();

866
867
	if (cpu_online(policy->cpu)) {
		down_write(&policy->rwsem);
868
		ret = fattr->store(policy, buf, count);
869
870
		up_write(&policy->rwsem);
	}
871

872
873
	put_online_cpus();

Linus Torvalds's avatar
Linus Torvalds committed
874
875
876
	return ret;
}

Dave Jones's avatar
Dave Jones committed
877
static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds's avatar
Linus Torvalds committed
878
{
Dave Jones's avatar
Dave Jones committed
879
	struct cpufreq_policy *policy = to_policy(kobj);
880
	pr_debug("last reference is dropped\n");
Linus Torvalds's avatar
Linus Torvalds committed
881
882
883
	complete(&policy->kobj_unregister);
}

884
static const struct sysfs_ops sysfs_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
885
886
887
888
889
890
891
892
893
894
	.show	= show,
	.store	= store,
};

static struct kobj_type ktype_cpufreq = {
	.sysfs_ops	= &sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= cpufreq_sysfs_release,
};

895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);

	if (!policy)
		return 0;

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return 0;

	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
}

static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return;

	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
}

/* Add/remove symlinks for all related CPUs */
925
static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
926
927
928
929
{
	unsigned int j;
	int ret = 0;

930
	/* Some related CPUs might not be present (physically hotplugged) */
931
	for_each_cpu(j, policy->real_cpus) {
932
		ret = add_cpu_dev_symlink(policy, j);
933
934
		if (ret)
			break;
935
	}
936

937
938
939
	return ret;
}

940
941
942
943
944
static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
{
	unsigned int j;

	/* Some related CPUs might not be present (physically hotplugged) */
945
	for_each_cpu(j, policy->real_cpus)
946
947
948
		remove_cpu_dev_symlink(policy, j);
}

949
static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
950
951
952
953
954
{
	struct freq_attr **drv_attr;
	int ret = 0;

	/* set up files for this cpu device */
955
	drv_attr = cpufreq_driver->attr;
956
	while (drv_attr && *drv_attr) {
957
958
		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
		if (ret)
959
			return ret;
960
961
		drv_attr++;
	}
962
	if (cpufreq_driver->get) {
963
964
		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
		if (ret)
965
			return ret;
966
	}
967
968
969

	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
	if (ret)
970
		return ret;
971

972
	if (cpufreq_driver->bios_limit) {
973
974
		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
		if (ret)
975
			return ret;
976
	}
977

978
	return cpufreq_add_dev_symlink(policy);
979
980
}

981
982
983
984
985
__weak struct cpufreq_governor *cpufreq_default_governor(void)
{
	return NULL;
}

986
static int cpufreq_init_policy(struct cpufreq_policy *policy)
987
{
988
	struct cpufreq_governor *gov = NULL;
989
990
	struct cpufreq_policy new_policy;

991
	memcpy(&new_policy, policy, sizeof(*policy));
992

993
	/* Update governor of new_policy to the governor used before hotplug */
994
	gov = find_governor(policy->last_governor);
995
	if (gov) {
996
997
		pr_debug("Restoring governor %s for cpu %d\n",
				policy->governor->name, policy->cpu);
998
999
1000
	} else {
		gov = cpufreq_default_governor();
		if (!gov)