cpufreq.c 66.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 *  linux/drivers/cpufreq/cpufreq.c
 *
 *  Copyright (C) 2001 Russell King
 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6
 *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds's avatar
Linus Torvalds committed
7
 *
8
 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9
 *	Added handling for CPU hotplug
10
11
 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
 *	Fix handling for CPU hotplug -- affected CPUs
12
 *
Linus Torvalds's avatar
Linus Torvalds committed
13
14
15
16
17
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

Viresh Kumar's avatar
Viresh Kumar committed
18
19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/cpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
23
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/device.h>
24
25
26
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
27
#include <linux/mutex.h>
28
#include <linux/slab.h>
29
#include <linux/suspend.h>
30
#include <linux/syscore_ops.h>
31
#include <linux/tick.h>
32
33
#include <trace/events/power.h>

34
static LIST_HEAD(cpufreq_policy_list);
35
36
37
38
39
40
41

static inline bool policy_is_inactive(struct cpufreq_policy *policy)
{
	return cpumask_empty(policy->cpus);
}

/* Macros to iterate over CPU policies */
42
43
44
#define for_each_suitable_policy(__policy, __active)			 \
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
		if ((__active) == !policy_is_inactive(__policy))
45
46
47
48
49
50
51

#define for_each_active_policy(__policy)		\
	for_each_suitable_policy(__policy, true)
#define for_each_inactive_policy(__policy)		\
	for_each_suitable_policy(__policy, false)

#define for_each_policy(__policy)			\
52
53
	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)

54
55
56
57
58
/* Iterate over governors */
static LIST_HEAD(cpufreq_governor_list);
#define for_each_governor(__governor)				\
	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)

Linus Torvalds's avatar
Linus Torvalds committed
59
/**
Dave Jones's avatar
Dave Jones committed
60
 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds's avatar
Linus Torvalds committed
61
62
63
 * level driver of CPUFreq support, and its spinlock. This lock
 * also protects the cpufreq_cpu_data array.
 */
64
static struct cpufreq_driver *cpufreq_driver;
65
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
66
67
static DEFINE_RWLOCK(cpufreq_driver_lock);

68
69
/* Flag to suspend/resume CPUFreq governors */
static bool cpufreq_suspended;
Linus Torvalds's avatar
Linus Torvalds committed
70

71
72
73
74
75
static inline bool has_target(void)
{
	return cpufreq_driver->target_index || cpufreq_driver->target;
}

Linus Torvalds's avatar
Linus Torvalds committed
76
/* internal prototypes */
77
static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
78
static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
79
static int cpufreq_start_governor(struct cpufreq_policy *policy);
80
81
82
83
84

static inline int cpufreq_exit_governor(struct cpufreq_policy *policy)
{
	return cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
}
Linus Torvalds's avatar
Linus Torvalds committed
85
86

/**
87
88
 * Two notifier lists: the "policy" list is involved in the
 * validation process for a new CPU frequency policy; the
Linus Torvalds's avatar
Linus Torvalds committed
89
90
91
92
 * "transition" list for kernel code that needs to handle
 * changes to devices when the CPU clock speed changes.
 * The mutex locks both lists.
 */
93
static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
94
static struct srcu_notifier_head cpufreq_transition_notifier_list;
Linus Torvalds's avatar
Linus Torvalds committed
95

96
static bool init_cpufreq_transition_notifier_list_called;
97
98
99
static int __init init_cpufreq_transition_notifier_list(void)
{
	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
100
	init_cpufreq_transition_notifier_list_called = true;
101
102
	return 0;
}
103
pure_initcall(init_cpufreq_transition_notifier_list);
Linus Torvalds's avatar
Linus Torvalds committed
104

105
static int off __read_mostly;
106
static int cpufreq_disabled(void)
107
108
109
110
111
112
113
{
	return off;
}
void disable_cpufreq(void)
{
	off = 1;
}
114
static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
115

116
117
bool have_governor_per_policy(void)
{
118
	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
119
}
120
EXPORT_SYMBOL_GPL(have_governor_per_policy);
121

122
123
124
125
126
127
128
129
130
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
{
	if (have_governor_per_policy())
		return &policy->kobj;
	else
		return cpufreq_global_kobject;
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);

131
132
133
134
135
136
137
138
139
struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

	return policy && !policy_is_inactive(policy) ?
		policy->freq_table : NULL;
}
EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
{
	u64 idle_time;
	u64 cur_wall_time;
	u64 busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());

	busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];

	idle_time = cur_wall_time - busy_time;
	if (wall)
		*wall = cputime_to_usecs(cur_wall_time);

	return cputime_to_usecs(idle_time);
}

u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);
	else if (!io_busy)
		idle_time += get_cpu_iowait_time_us(cpu, wall);

	return idle_time;
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);

175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
/*
 * This is a generic cpufreq init() routine which can be used by cpufreq
 * drivers of SMP systems. It will do following:
 * - validate & show freq table passed
 * - set policies transition latency
 * - policy->cpus with all possible CPUs
 */
int cpufreq_generic_init(struct cpufreq_policy *policy,
		struct cpufreq_frequency_table *table,
		unsigned int transition_latency)
{
	int ret;

	ret = cpufreq_table_validate_and_show(policy, table);
	if (ret) {
		pr_err("%s: invalid frequency table: %d\n", __func__, ret);
		return ret;
	}

	policy->cpuinfo.transition_latency = transition_latency;

	/*
197
	 * The driver only supports the SMP configuration where all processors
198
199
200
201
202
203
204
205
	 * share the clock and voltage and clock.
	 */
	cpumask_setall(policy->cpus);

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);

206
struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
207
208
209
{
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);

210
211
	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
}
212
EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
213
214
215
216
217

unsigned int cpufreq_generic_get(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);

218
	if (!policy || IS_ERR(policy->clk)) {
219
220
		pr_err("%s: No %s associated to cpu: %d\n",
		       __func__, policy ? "clk" : "policy", cpu);
221
222
223
224
225
226
227
		return 0;
	}

	return clk_get_rate(policy->clk) / 1000;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_get);

228
229
230
231
232
233
234
235
236
237
238
239
240
/**
 * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 *
 * @cpu: cpu to find policy for.
 *
 * This returns policy for 'cpu', returns NULL if it doesn't exist.
 * It also increments the kobject reference count to mark it busy and so would
 * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 * freed as that depends on the kobj count.
 *
 * Return: A valid policy on success, otherwise NULL on failure.
 */
241
struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
242
{
243
	struct cpufreq_policy *policy = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
244
245
	unsigned long flags;

246
	if (WARN_ON(cpu >= nr_cpu_ids))
247
248
		return NULL;

Linus Torvalds's avatar
Linus Torvalds committed
249
	/* get the cpufreq driver */
250
	read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
251

252
253
	if (cpufreq_driver) {
		/* get the CPU */
254
		policy = cpufreq_cpu_get_raw(cpu);
255
256
257
		if (policy)
			kobject_get(&policy->kobj);
	}
Linus Torvalds's avatar
Linus Torvalds committed
258

259
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
260

261
	return policy;
262
}
Linus Torvalds's avatar
Linus Torvalds committed
263
264
EXPORT_SYMBOL_GPL(cpufreq_cpu_get);

265
266
267
268
269
270
271
272
/**
 * cpufreq_cpu_put: Decrements the usage count of a policy
 *
 * @policy: policy earlier returned by cpufreq_cpu_get().
 *
 * This decrements the kobject reference count incremented earlier by calling
 * cpufreq_cpu_get().
 */
273
void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
274
{
275
	kobject_put(&policy->kobj);
Linus Torvalds's avatar
Linus Torvalds committed
276
277
278
279
280
281
282
283
284
285
286
287
}
EXPORT_SYMBOL_GPL(cpufreq_cpu_put);

/*********************************************************************
 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 *********************************************************************/

/**
 * adjust_jiffies - adjust the system "loops_per_jiffy"
 *
 * This function alters the system "loops_per_jiffy" for the clock
 * speed change. Note that loops_per_jiffy cannot be updated on SMP
288
 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds's avatar
Linus Torvalds committed
289
290
 * per-CPU loops_per_jiffy value wherever possible.
 */
291
static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds's avatar
Linus Torvalds committed
292
{
293
294
295
296
#ifndef CONFIG_SMP
	static unsigned long l_p_j_ref;
	static unsigned int l_p_j_ref_freq;

Linus Torvalds's avatar
Linus Torvalds committed
297
298
299
300
301
302
	if (ci->flags & CPUFREQ_CONST_LOOPS)
		return;

	if (!l_p_j_ref_freq) {
		l_p_j_ref = loops_per_jiffy;
		l_p_j_ref_freq = ci->old;
303
304
		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
			 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds's avatar
Linus Torvalds committed
305
	}
306
	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
307
308
		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
								ci->new);
309
310
		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
			 loops_per_jiffy, ci->new);
Linus Torvalds's avatar
Linus Torvalds committed
311
312
	}
#endif
313
}
Linus Torvalds's avatar
Linus Torvalds committed
314

315
static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
316
		struct cpufreq_freqs *freqs, unsigned int state)
Linus Torvalds's avatar
Linus Torvalds committed
317
318
319
{
	BUG_ON(irqs_disabled());

320
321
322
	if (cpufreq_disabled())
		return;

323
	freqs->flags = cpufreq_driver->flags;
324
	pr_debug("notification %u of frequency transition to %u kHz\n",
325
		 state, freqs->new);
Linus Torvalds's avatar
Linus Torvalds committed
326
327

	switch (state) {
328

Linus Torvalds's avatar
Linus Torvalds committed
329
	case CPUFREQ_PRECHANGE:
330
		/* detect if the driver reported a value as "old frequency"
331
332
		 * which is not equal to what the cpufreq core thinks is
		 * "old frequency".
Linus Torvalds's avatar
Linus Torvalds committed
333
		 */
334
		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
335
336
			if ((policy) && (policy->cpu == freqs->cpu) &&
			    (policy->cur) && (policy->cur != freqs->old)) {
337
338
				pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
					 freqs->old, policy->cur);
339
				freqs->old = policy->cur;
Linus Torvalds's avatar
Linus Torvalds committed
340
341
			}
		}
342
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
343
				CPUFREQ_PRECHANGE, freqs);
Linus Torvalds's avatar
Linus Torvalds committed
344
345
		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
		break;
346

Linus Torvalds's avatar
Linus Torvalds committed
347
348
	case CPUFREQ_POSTCHANGE:
		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
349
350
		pr_debug("FREQ: %lu - CPU: %lu\n",
			 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
351
		trace_cpu_frequency(freqs->new, freqs->cpu);
352
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
353
				CPUFREQ_POSTCHANGE, freqs);
354
355
		if (likely(policy) && likely(policy->cpu == freqs->cpu))
			policy->cur = freqs->new;
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
		break;
	}
}
359

360
361
362
363
364
365
366
367
/**
 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 * on frequency transition.
 *
 * This function calls the transition notifiers and the "adjust_jiffies"
 * function. It is called twice on all CPU frequency changes that have
 * external effects.
 */
368
static void cpufreq_notify_transition(struct cpufreq_policy *policy,
369
370
371
372
373
		struct cpufreq_freqs *freqs, unsigned int state)
{
	for_each_cpu(freqs->cpu, policy->cpus)
		__cpufreq_notify_transition(policy, freqs, state);
}
Linus Torvalds's avatar
Linus Torvalds committed
374

375
/* Do post notifications when there are chances that transition has failed */
376
static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
377
378
379
380
381
382
383
384
385
386
387
		struct cpufreq_freqs *freqs, int transition_failed)
{
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
	if (!transition_failed)
		return;

	swap(freqs->old, freqs->new);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
}

388
389
390
void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs)
{
391
392
393
394
395
396
397
398
399
400
401
402

	/*
	 * Catch double invocations of _begin() which lead to self-deadlock.
	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
	 * doesn't invoke _begin() on their behalf, and hence the chances of
	 * double invocations are very low. Moreover, there are scenarios
	 * where these checks can emit false-positive warnings in these
	 * drivers; so we avoid that by skipping them altogether.
	 */
	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
				&& current == policy->transition_task);

403
404
405
406
407
408
409
410
411
412
413
wait:
	wait_event(policy->transition_wait, !policy->transition_ongoing);

	spin_lock(&policy->transition_lock);

	if (unlikely(policy->transition_ongoing)) {
		spin_unlock(&policy->transition_lock);
		goto wait;
	}

	policy->transition_ongoing = true;
414
	policy->transition_task = current;
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430

	spin_unlock(&policy->transition_lock);

	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);

void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs, int transition_failed)
{
	if (unlikely(WARN_ON(!policy->transition_ongoing)))
		return;

	cpufreq_notify_post_transition(policy, freqs, transition_failed);

	policy->transition_ongoing = false;
431
	policy->transition_task = NULL;
432
433
434
435
436

	wake_up(&policy->transition_wait);
}
EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
/*
 * Fast frequency switching status count.  Positive means "enabled", negative
 * means "disabled" and 0 means "not decided yet".
 */
static int cpufreq_fast_switch_count;
static DEFINE_MUTEX(cpufreq_fast_switch_lock);

static void cpufreq_list_transition_notifiers(void)
{
	struct notifier_block *nb;

	pr_info("Registered transition notifiers:\n");

	mutex_lock(&cpufreq_transition_notifier_list.mutex);

	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
		pr_info("%pF\n", nb->notifier_call);

	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
}

/**
 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
 * @policy: cpufreq policy to enable fast frequency switching for.
 *
 * Try to enable fast frequency switching for @policy.
 *
 * The attempt will fail if there is at least one transition notifier registered
 * at this point, as fast frequency switching is quite fundamentally at odds
 * with transition notifiers.  Thus if successful, it will make registration of
 * transition notifiers fail going forward.
 */
void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
{
	lockdep_assert_held(&policy->rwsem);

	if (!policy->fast_switch_possible)
		return;

	mutex_lock(&cpufreq_fast_switch_lock);
	if (cpufreq_fast_switch_count >= 0) {
		cpufreq_fast_switch_count++;
		policy->fast_switch_enabled = true;
	} else {
		pr_warn("CPU%u: Fast frequency switching not enabled\n",
			policy->cpu);
		cpufreq_list_transition_notifiers();
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);

489
490
491
492
493
/**
 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
 * @policy: cpufreq policy to disable fast frequency switching for.
 */
void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
494
495
496
497
498
499
500
501
502
{
	mutex_lock(&cpufreq_fast_switch_lock);
	if (policy->fast_switch_enabled) {
		policy->fast_switch_enabled = false;
		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
			cpufreq_fast_switch_count--;
	}
	mutex_unlock(&cpufreq_fast_switch_lock);
}
503
EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
Linus Torvalds's avatar
Linus Torvalds committed
504
505
506
507

/*********************************************************************
 *                          SYSFS INTERFACE                          *
 *********************************************************************/
508
static ssize_t show_boost(struct kobject *kobj,
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
				 struct attribute *attr, char *buf)
{
	return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
}

static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
				  const char *buf, size_t count)
{
	int ret, enable;

	ret = sscanf(buf, "%d", &enable);
	if (ret != 1 || enable < 0 || enable > 1)
		return -EINVAL;

	if (cpufreq_boost_trigger_state(enable)) {
524
525
		pr_err("%s: Cannot %s BOOST!\n",
		       __func__, enable ? "enable" : "disable");
526
527
528
		return -EINVAL;
	}

529
530
	pr_debug("%s: cpufreq BOOST %s\n",
		 __func__, enable ? "enabled" : "disabled");
531
532
533
534

	return count;
}
define_one_global_rw(boost);
Linus Torvalds's avatar
Linus Torvalds committed
535

536
static struct cpufreq_governor *find_governor(const char *str_governor)
537
538
539
{
	struct cpufreq_governor *t;

540
	for_each_governor(t)
541
		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
542
543
544
545
546
			return t;

	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
547
548
549
/**
 * cpufreq_parse_governor - parse a governor string
 */
Dave Jones's avatar
Dave Jones committed
550
static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
Linus Torvalds's avatar
Linus Torvalds committed
551
552
				struct cpufreq_governor **governor)
{
553
	int err = -EINVAL;
554
555

	if (cpufreq_driver->setpolicy) {
556
		if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
557
			*policy = CPUFREQ_POLICY_PERFORMANCE;
558
			err = 0;
559
		} else if (!strncasecmp(str_governor, "powersave",
560
						CPUFREQ_NAME_LEN)) {
Linus Torvalds's avatar
Linus Torvalds committed
561
			*policy = CPUFREQ_POLICY_POWERSAVE;
562
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
563
		}
564
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
565
		struct cpufreq_governor *t;
566

567
		mutex_lock(&cpufreq_governor_mutex);
568

569
		t = find_governor(str_governor);
570

571
		if (t == NULL) {
572
			int ret;
573

574
575
576
			mutex_unlock(&cpufreq_governor_mutex);
			ret = request_module("cpufreq_%s", str_governor);
			mutex_lock(&cpufreq_governor_mutex);
577

578
			if (ret == 0)
579
				t = find_governor(str_governor);
580
581
		}

582
583
584
		if (t != NULL) {
			*governor = t;
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
585
		}
586

587
		mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
588
	}
589
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
590
591
592
}

/**
593
594
 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 * print out cpufreq information
Linus Torvalds's avatar
Linus Torvalds committed
595
596
597
598
599
 *
 * Write out information from cpufreq_driver->policy[cpu]; object must be
 * "unsigned int".
 */

600
601
#define show_one(file_name, object)			\
static ssize_t show_##file_name				\
Dave Jones's avatar
Dave Jones committed
602
(struct cpufreq_policy *policy, char *buf)		\
603
{							\
604
	return sprintf(buf, "%u\n", policy->object);	\
Linus Torvalds's avatar
Linus Torvalds committed
605
606
607
608
}

show_one(cpuinfo_min_freq, cpuinfo.min_freq);
show_one(cpuinfo_max_freq, cpuinfo.max_freq);
609
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds's avatar
Linus Torvalds committed
610
611
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
612

613
static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
614
615
616
617
618
619
620
621
622
{
	ssize_t ret;

	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
	else
		ret = sprintf(buf, "%u\n", policy->cur);
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
623

624
static int cpufreq_set_policy(struct cpufreq_policy *policy,
625
				struct cpufreq_policy *new_policy);
626

Linus Torvalds's avatar
Linus Torvalds committed
627
628
629
630
631
/**
 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 */
#define store_one(file_name, object)			\
static ssize_t store_##file_name					\
Dave Jones's avatar
Dave Jones committed
632
(struct cpufreq_policy *policy, const char *buf, size_t count)		\
Linus Torvalds's avatar
Linus Torvalds committed
633
{									\
634
	int ret, temp;							\
Linus Torvalds's avatar
Linus Torvalds committed
635
636
	struct cpufreq_policy new_policy;				\
									\
637
	memcpy(&new_policy, policy, sizeof(*policy));			\
Linus Torvalds's avatar
Linus Torvalds committed
638
									\
639
	ret = sscanf(buf, "%u", &new_policy.object);			\
Linus Torvalds's avatar
Linus Torvalds committed
640
641
642
	if (ret != 1)							\
		return -EINVAL;						\
									\
643
	temp = new_policy.object;					\
644
	ret = cpufreq_set_policy(policy, &new_policy);		\
645
646
	if (!ret)							\
		policy->user_policy.object = temp;			\
Linus Torvalds's avatar
Linus Torvalds committed
647
648
649
650
									\
	return ret ? ret : count;					\
}

651
652
store_one(scaling_min_freq, min);
store_one(scaling_max_freq, max);
Linus Torvalds's avatar
Linus Torvalds committed
653
654
655
656

/**
 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 */
Dave Jones's avatar
Dave Jones committed
657
658
static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
					char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
659
{
660
	unsigned int cur_freq = __cpufreq_get(policy);
Linus Torvalds's avatar
Linus Torvalds committed
661
662
663
664
665
666
667
668
	if (!cur_freq)
		return sprintf(buf, "<unknown>");
	return sprintf(buf, "%u\n", cur_freq);
}

/**
 * show_scaling_governor - show the current policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
669
static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
670
{
671
	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds's avatar
Linus Torvalds committed
672
673
674
675
		return sprintf(buf, "powersave\n");
	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
		return sprintf(buf, "performance\n");
	else if (policy->governor)
676
		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
677
				policy->governor->name);
Linus Torvalds's avatar
Linus Torvalds committed
678
679
680
681
682
683
	return -EINVAL;
}

/**
 * store_scaling_governor - store policy for the specified CPU
 */
Dave Jones's avatar
Dave Jones committed
684
685
static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
					const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
686
{
687
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
688
689
690
	char	str_governor[16];
	struct cpufreq_policy new_policy;

691
	memcpy(&new_policy, policy, sizeof(*policy));
Linus Torvalds's avatar
Linus Torvalds committed
692

693
	ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds's avatar
Linus Torvalds committed
694
695
696
	if (ret != 1)
		return -EINVAL;

697
698
	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
						&new_policy.governor))
Linus Torvalds's avatar
Linus Torvalds committed
699
700
		return -EINVAL;

701
	ret = cpufreq_set_policy(policy, &new_policy);
702
	return ret ? ret : count;
Linus Torvalds's avatar
Linus Torvalds committed
703
704
705
706
707
}

/**
 * show_scaling_driver - show the cpufreq driver currently loaded
 */
Dave Jones's avatar
Dave Jones committed
708
static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
709
{
710
	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds's avatar
Linus Torvalds committed
711
712
713
714
715
}

/**
 * show_scaling_available_governors - show the available CPUfreq governors
 */
Dave Jones's avatar
Dave Jones committed
716
717
static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
						char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
718
719
720
721
{
	ssize_t i = 0;
	struct cpufreq_governor *t;

722
	if (!has_target()) {
Linus Torvalds's avatar
Linus Torvalds committed
723
724
725
726
		i += sprintf(buf, "performance powersave");
		goto out;
	}

727
	for_each_governor(t) {
728
729
		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
		    - (CPUFREQ_NAME_LEN + 2)))
Linus Torvalds's avatar
Linus Torvalds committed
730
			goto out;
731
		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds's avatar
Linus Torvalds committed
732
	}
733
out:
Linus Torvalds's avatar
Linus Torvalds committed
734
735
736
	i += sprintf(&buf[i], "\n");
	return i;
}
737

738
ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
739
740
741
742
{
	ssize_t i = 0;
	unsigned int cpu;

743
	for_each_cpu(cpu, mask) {
Linus Torvalds's avatar
Linus Torvalds committed
744
745
746
747
		if (i)
			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
		if (i >= (PAGE_SIZE - 5))
748
			break;
Linus Torvalds's avatar
Linus Torvalds committed
749
750
751
752
	}
	i += sprintf(&buf[i], "\n");
	return i;
}
753
EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds's avatar
Linus Torvalds committed
754

755
756
757
758
759
760
/**
 * show_related_cpus - show the CPUs affected by each transition even if
 * hw coordination is in use
 */
static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
{
761
	return cpufreq_show_cpus(policy->related_cpus, buf);
762
763
764
765
766
767
768
}

/**
 * show_affected_cpus - show the CPUs affected by each transition
 */
static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
{
769
	return cpufreq_show_cpus(policy->cpus, buf);
770
771
}

772
static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones's avatar
Dave Jones committed
773
					const char *buf, size_t count)
774
775
776
777
{
	unsigned int freq = 0;
	unsigned int ret;

778
	if (!policy->governor || !policy->governor->store_setspeed)
779
780
781
782
783
784
785
786
787
788
789
790
791
		return -EINVAL;

	ret = sscanf(buf, "%u", &freq);
	if (ret != 1)
		return -EINVAL;

	policy->governor->store_setspeed(policy, freq);

	return count;
}

static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
{
792
	if (!policy->governor || !policy->governor->show_setspeed)
793
794
795
796
		return sprintf(buf, "<unsupported>\n");

	return policy->governor->show_setspeed(policy, buf);
}
Linus Torvalds's avatar
Linus Torvalds committed
797

798
/**
799
 * show_bios_limit - show the current cpufreq HW/BIOS limitation
800
801
802
803
804
 */
static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
{
	unsigned int limit;
	int ret;
805
806
	if (cpufreq_driver->bios_limit) {
		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
807
808
809
810
811
812
		if (!ret)
			return sprintf(buf, "%u\n", limit);
	}
	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
}

813
814
815
816
817
818
819
820
821
822
823
824
825
826
cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
cpufreq_freq_attr_ro(cpuinfo_min_freq);
cpufreq_freq_attr_ro(cpuinfo_max_freq);
cpufreq_freq_attr_ro(cpuinfo_transition_latency);
cpufreq_freq_attr_ro(scaling_available_governors);
cpufreq_freq_attr_ro(scaling_driver);
cpufreq_freq_attr_ro(scaling_cur_freq);
cpufreq_freq_attr_ro(bios_limit);
cpufreq_freq_attr_ro(related_cpus);
cpufreq_freq_attr_ro(affected_cpus);
cpufreq_freq_attr_rw(scaling_min_freq);
cpufreq_freq_attr_rw(scaling_max_freq);
cpufreq_freq_attr_rw(scaling_governor);
cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds's avatar
Linus Torvalds committed
827

Dave Jones's avatar
Dave Jones committed
828
static struct attribute *default_attrs[] = {
Linus Torvalds's avatar
Linus Torvalds committed
829
830
	&cpuinfo_min_freq.attr,
	&cpuinfo_max_freq.attr,
831
	&cpuinfo_transition_latency.attr,
Linus Torvalds's avatar
Linus Torvalds committed
832
833
834
	&scaling_min_freq.attr,
	&scaling_max_freq.attr,
	&affected_cpus.attr,
835
	&related_cpus.attr,
Linus Torvalds's avatar
Linus Torvalds committed
836
837
838
	&scaling_governor.attr,
	&scaling_driver.attr,
	&scaling_available_governors.attr,
839
	&scaling_setspeed.attr,
Linus Torvalds's avatar
Linus Torvalds committed
840
841
842
	NULL
};

843
844
#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds's avatar
Linus Torvalds committed
845

846
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds's avatar
Linus Torvalds committed
847
{
Dave Jones's avatar
Dave Jones committed
848
849
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
850
	ssize_t ret;
851

852
	down_read(&policy->rwsem);
853
	ret = fattr->show(policy, buf);
854
	up_read(&policy->rwsem);
855

Linus Torvalds's avatar
Linus Torvalds committed
856
857
858
	return ret;
}

Dave Jones's avatar
Dave Jones committed
859
860
static ssize_t store(struct kobject *kobj, struct attribute *attr,
		     const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
861
{
Dave Jones's avatar
Dave Jones committed
862
863
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
864
	ssize_t ret = -EINVAL;
865

866
867
	get_online_cpus();

868
869
	if (cpu_online(policy->cpu)) {
		down_write(&policy->rwsem);
870
		ret = fattr->store(policy, buf, count);
871
872
		up_write(&policy->rwsem);
	}
873

874
875
	put_online_cpus();

Linus Torvalds's avatar
Linus Torvalds committed
876
877
878
	return ret;
}

Dave Jones's avatar
Dave Jones committed
879
static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds's avatar
Linus Torvalds committed
880
{
Dave Jones's avatar
Dave Jones committed
881
	struct cpufreq_policy *policy = to_policy(kobj);
882
	pr_debug("last reference is dropped\n");
Linus Torvalds's avatar
Linus Torvalds committed
883
884
885
	complete(&policy->kobj_unregister);
}

886
static const struct sysfs_ops sysfs_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
887
888
889
890
891
892
893
894
895
896
	.show	= show,
	.store	= store,
};

static struct kobj_type ktype_cpufreq = {
	.sysfs_ops	= &sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= cpufreq_sysfs_release,
};

897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);

	if (!policy)
		return 0;

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return 0;

	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
}

static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
{
	struct device *cpu_dev;

	pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);

	cpu_dev = get_cpu_device(cpu);
	if (WARN_ON(!cpu_dev))
		return;

	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
}

/* Add/remove symlinks for all related CPUs */
927
static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
928
929
930
931
{
	unsigned int j;
	int ret = 0;

932
	/* Some related CPUs might not be present (physically hotplugged) */
933
	for_each_cpu(j, policy->real_cpus) {
934
		ret = add_cpu_dev_symlink(policy, j);
935
936
		if (ret)
			break;
937
	}
938

939
940
941
	return ret;
}

942
943
944
945
946
static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
{
	unsigned int j;

	/* Some related CPUs might not be present (physically hotplugged) */
947
	for_each_cpu(j, policy->real_cpus)
948
949
950
		remove_cpu_dev_symlink(policy, j);
}

951
static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
952
953
954
955
956
{
	struct freq_attr **drv_attr;
	int ret = 0;

	/* set up files for this cpu device */
957
	drv_attr = cpufreq_driver->attr;
958
	while (drv_attr && *drv_attr) {
959
960
		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
		if (ret)
961
			return ret;
962
963
		drv_attr++;
	}
964
	if (cpufreq_driver->get) {
965
966
		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
		if (ret)
967
			return ret;
968
	}
969
970
971

	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
	if (ret)
972
		return ret;
973

974
	if (cpufreq_driver->bios_limit) {
975
976
		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
		if (ret)
977
			return ret;
978
	}
979

980
	return cpufreq_add_dev_symlink(policy);
981
982
}

983
984
985
986
987
__weak struct cpufreq_governor *cpufreq_default_governor(void)
{
	return NULL;
}

988
static int cpufreq_init_policy(struct cpufreq_policy *policy)
989
{
990
	struct cpufreq_governor *gov = NULL;
991
992
	struct cpufreq_policy new_policy;

993
	memcpy(&new_policy, policy, sizeof(*policy));
994

995
	/* Update governor of new_policy to the governor used before hotplug */
996
	gov = find_governor(policy->last_governor);
997
	if (gov) {
998
999
		pr_debug("Restoring governor %s for cpu %d\n",
				policy->governor->name, policy->cpu);
1000
	} else {