1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/drivers/cpufreq/cpufreq.c
4  *
5  *  Copyright (C) 2001 Russell King
6  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8  *
9  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10  *	Added handling for CPU hotplug
11  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12  *	Fix handling for CPU hotplug -- affected CPUs
13  */
14 
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16 
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/cpu_cooling.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/init.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/pm_qos.h>
27 #include <linux/slab.h>
28 #include <linux/suspend.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/tick.h>
31 #include <linux/units.h>
32 #include <trace/events/power.h>
33 
34 static LIST_HEAD(cpufreq_policy_list);
35 
36 /* Macros to iterate over CPU policies */
37 #define for_each_suitable_policy(__policy, __active)			 \
38 	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
39 		if ((__active) == !policy_is_inactive(__policy))
40 
41 #define for_each_active_policy(__policy)		\
42 	for_each_suitable_policy(__policy, true)
43 #define for_each_inactive_policy(__policy)		\
44 	for_each_suitable_policy(__policy, false)
45 
46 /* Iterate over governors */
47 static LIST_HEAD(cpufreq_governor_list);
48 #define for_each_governor(__governor)				\
49 	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
50 
51 static char default_governor[CPUFREQ_NAME_LEN];
52 
53 /*
54  * The "cpufreq driver" - the arch- or hardware-dependent low
55  * level driver of CPUFreq support, and its spinlock. This lock
56  * also protects the cpufreq_cpu_data array.
57  */
58 static struct cpufreq_driver *cpufreq_driver;
59 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
60 static DEFINE_RWLOCK(cpufreq_driver_lock);
61 
62 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
cpufreq_supports_freq_invariance(void)63 bool cpufreq_supports_freq_invariance(void)
64 {
65 	return static_branch_likely(&cpufreq_freq_invariance);
66 }
67 
68 /* Flag to suspend/resume CPUFreq governors */
69 static bool cpufreq_suspended;
70 
has_target(void)71 static inline bool has_target(void)
72 {
73 	return cpufreq_driver->target_index || cpufreq_driver->target;
74 }
75 
has_target_index(void)76 bool has_target_index(void)
77 {
78 	return !!cpufreq_driver->target_index;
79 }
80 
81 /* internal prototypes */
82 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
83 static int cpufreq_init_governor(struct cpufreq_policy *policy);
84 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
85 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
86 static int cpufreq_set_policy(struct cpufreq_policy *policy,
87 			      struct cpufreq_governor *new_gov,
88 			      unsigned int new_pol);
89 static bool cpufreq_boost_supported(void);
90 
91 /*
92  * Two notifier lists: the "policy" list is involved in the
93  * validation process for a new CPU frequency policy; the
94  * "transition" list for kernel code that needs to handle
95  * changes to devices when the CPU clock speed changes.
96  * The mutex locks both lists.
97  */
98 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
99 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
100 
101 static int off __read_mostly;
cpufreq_disabled(void)102 static int cpufreq_disabled(void)
103 {
104 	return off;
105 }
disable_cpufreq(void)106 void disable_cpufreq(void)
107 {
108 	off = 1;
109 }
110 static DEFINE_MUTEX(cpufreq_governor_mutex);
111 
have_governor_per_policy(void)112 bool have_governor_per_policy(void)
113 {
114 	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
115 }
116 EXPORT_SYMBOL_GPL(have_governor_per_policy);
117 
118 static struct kobject *cpufreq_global_kobject;
119 
get_governor_parent_kobj(struct cpufreq_policy * policy)120 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
121 {
122 	if (have_governor_per_policy())
123 		return &policy->kobj;
124 	else
125 		return cpufreq_global_kobject;
126 }
127 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
128 
get_cpu_idle_time_jiffy(unsigned int cpu,u64 * wall)129 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
130 {
131 	struct kernel_cpustat kcpustat;
132 	u64 cur_wall_time;
133 	u64 idle_time;
134 	u64 busy_time;
135 
136 	cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
137 
138 	kcpustat_cpu_fetch(&kcpustat, cpu);
139 
140 	busy_time = kcpustat.cpustat[CPUTIME_USER];
141 	busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
142 	busy_time += kcpustat.cpustat[CPUTIME_IRQ];
143 	busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
144 	busy_time += kcpustat.cpustat[CPUTIME_STEAL];
145 	busy_time += kcpustat.cpustat[CPUTIME_NICE];
146 
147 	idle_time = cur_wall_time - busy_time;
148 	if (wall)
149 		*wall = div_u64(cur_wall_time, NSEC_PER_USEC);
150 
151 	return div_u64(idle_time, NSEC_PER_USEC);
152 }
153 
get_cpu_idle_time(unsigned int cpu,u64 * wall,int io_busy)154 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
155 {
156 	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
157 
158 	if (idle_time == -1ULL)
159 		return get_cpu_idle_time_jiffy(cpu, wall);
160 	else if (!io_busy)
161 		idle_time += get_cpu_iowait_time_us(cpu, wall);
162 
163 	return idle_time;
164 }
165 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
166 
167 /*
168  * This is a generic cpufreq init() routine which can be used by cpufreq
169  * drivers of SMP systems. It will do following:
170  * - validate & show freq table passed
171  * - set policies transition latency
172  * - policy->cpus with all possible CPUs
173  */
cpufreq_generic_init(struct cpufreq_policy * policy,struct cpufreq_frequency_table * table,unsigned int transition_latency)174 void cpufreq_generic_init(struct cpufreq_policy *policy,
175 		struct cpufreq_frequency_table *table,
176 		unsigned int transition_latency)
177 {
178 	policy->freq_table = table;
179 	policy->cpuinfo.transition_latency = transition_latency;
180 
181 	/*
182 	 * The driver only supports the SMP configuration where all processors
183 	 * share the clock and voltage and clock.
184 	 */
185 	cpumask_setall(policy->cpus);
186 }
187 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
188 
cpufreq_cpu_get_raw(unsigned int cpu)189 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
190 {
191 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
192 
193 	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
194 }
195 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
196 
cpufreq_generic_get(unsigned int cpu)197 unsigned int cpufreq_generic_get(unsigned int cpu)
198 {
199 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
200 
201 	if (!policy || IS_ERR(policy->clk)) {
202 		pr_err("%s: No %s associated to cpu: %d\n",
203 		       __func__, policy ? "clk" : "policy", cpu);
204 		return 0;
205 	}
206 
207 	return clk_get_rate(policy->clk) / 1000;
208 }
209 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
210 
211 /**
212  * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
213  * @cpu: CPU to find the policy for.
214  *
215  * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
216  * the kobject reference counter of that policy.  Return a valid policy on
217  * success or NULL on failure.
218  *
219  * The policy returned by this function has to be released with the help of
220  * cpufreq_cpu_put() to balance its kobject reference counter properly.
221  */
cpufreq_cpu_get(unsigned int cpu)222 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
223 {
224 	struct cpufreq_policy *policy = NULL;
225 	unsigned long flags;
226 
227 	if (WARN_ON(cpu >= nr_cpu_ids))
228 		return NULL;
229 
230 	/* get the cpufreq driver */
231 	read_lock_irqsave(&cpufreq_driver_lock, flags);
232 
233 	if (cpufreq_driver) {
234 		/* get the CPU */
235 		policy = cpufreq_cpu_get_raw(cpu);
236 		if (policy)
237 			kobject_get(&policy->kobj);
238 	}
239 
240 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
241 
242 	return policy;
243 }
244 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
245 
246 /**
247  * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
248  * @policy: cpufreq policy returned by cpufreq_cpu_get().
249  */
cpufreq_cpu_put(struct cpufreq_policy * policy)250 void cpufreq_cpu_put(struct cpufreq_policy *policy)
251 {
252 	kobject_put(&policy->kobj);
253 }
254 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
255 
256 /**
257  * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
258  * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
259  */
cpufreq_cpu_release(struct cpufreq_policy * policy)260 void cpufreq_cpu_release(struct cpufreq_policy *policy)
261 {
262 	if (WARN_ON(!policy))
263 		return;
264 
265 	lockdep_assert_held(&policy->rwsem);
266 
267 	up_write(&policy->rwsem);
268 
269 	cpufreq_cpu_put(policy);
270 }
271 
272 /**
273  * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
274  * @cpu: CPU to find the policy for.
275  *
276  * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
277  * if the policy returned by it is not NULL, acquire its rwsem for writing.
278  * Return the policy if it is active or release it and return NULL otherwise.
279  *
280  * The policy returned by this function has to be released with the help of
281  * cpufreq_cpu_release() in order to release its rwsem and balance its usage
282  * counter properly.
283  */
cpufreq_cpu_acquire(unsigned int cpu)284 struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
285 {
286 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
287 
288 	if (!policy)
289 		return NULL;
290 
291 	down_write(&policy->rwsem);
292 
293 	if (policy_is_inactive(policy)) {
294 		cpufreq_cpu_release(policy);
295 		return NULL;
296 	}
297 
298 	return policy;
299 }
300 
301 /*********************************************************************
302  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
303  *********************************************************************/
304 
305 /**
306  * adjust_jiffies - Adjust the system "loops_per_jiffy".
307  * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
308  * @ci: Frequency change information.
309  *
310  * This function alters the system "loops_per_jiffy" for the clock
311  * speed change. Note that loops_per_jiffy cannot be updated on SMP
312  * systems as each CPU might be scaled differently. So, use the arch
313  * per-CPU loops_per_jiffy value wherever possible.
314  */
adjust_jiffies(unsigned long val,struct cpufreq_freqs * ci)315 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
316 {
317 #ifndef CONFIG_SMP
318 	static unsigned long l_p_j_ref;
319 	static unsigned int l_p_j_ref_freq;
320 
321 	if (ci->flags & CPUFREQ_CONST_LOOPS)
322 		return;
323 
324 	if (!l_p_j_ref_freq) {
325 		l_p_j_ref = loops_per_jiffy;
326 		l_p_j_ref_freq = ci->old;
327 		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
328 			 l_p_j_ref, l_p_j_ref_freq);
329 	}
330 	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
331 		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
332 								ci->new);
333 		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
334 			 loops_per_jiffy, ci->new);
335 	}
336 #endif
337 }
338 
339 /**
340  * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
341  * @policy: cpufreq policy to enable fast frequency switching for.
342  * @freqs: contain details of the frequency update.
343  * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
344  *
345  * This function calls the transition notifiers and adjust_jiffies().
346  *
347  * It is called twice on all CPU frequency changes that have external effects.
348  */
cpufreq_notify_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,unsigned int state)349 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
350 				      struct cpufreq_freqs *freqs,
351 				      unsigned int state)
352 {
353 	int cpu;
354 
355 	BUG_ON(irqs_disabled());
356 
357 	if (cpufreq_disabled())
358 		return;
359 
360 	freqs->policy = policy;
361 	freqs->flags = cpufreq_driver->flags;
362 	pr_debug("notification %u of frequency transition to %u kHz\n",
363 		 state, freqs->new);
364 
365 	switch (state) {
366 	case CPUFREQ_PRECHANGE:
367 		/*
368 		 * Detect if the driver reported a value as "old frequency"
369 		 * which is not equal to what the cpufreq core thinks is
370 		 * "old frequency".
371 		 */
372 		if (policy->cur && policy->cur != freqs->old) {
373 			pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
374 				 freqs->old, policy->cur);
375 			freqs->old = policy->cur;
376 		}
377 
378 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
379 					 CPUFREQ_PRECHANGE, freqs);
380 
381 		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
382 		break;
383 
384 	case CPUFREQ_POSTCHANGE:
385 		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
386 		pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
387 			 cpumask_pr_args(policy->cpus));
388 
389 		for_each_cpu(cpu, policy->cpus)
390 			trace_cpu_frequency(freqs->new, cpu);
391 
392 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
393 					 CPUFREQ_POSTCHANGE, freqs);
394 
395 		cpufreq_stats_record_transition(policy, freqs->new);
396 		policy->cur = freqs->new;
397 	}
398 }
399 
400 /* Do post notifications when there are chances that transition has failed */
cpufreq_notify_post_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)401 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
402 		struct cpufreq_freqs *freqs, int transition_failed)
403 {
404 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
405 	if (!transition_failed)
406 		return;
407 
408 	swap(freqs->old, freqs->new);
409 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
410 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
411 }
412 
cpufreq_freq_transition_begin(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs)413 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
414 		struct cpufreq_freqs *freqs)
415 {
416 
417 	/*
418 	 * Catch double invocations of _begin() which lead to self-deadlock.
419 	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
420 	 * doesn't invoke _begin() on their behalf, and hence the chances of
421 	 * double invocations are very low. Moreover, there are scenarios
422 	 * where these checks can emit false-positive warnings in these
423 	 * drivers; so we avoid that by skipping them altogether.
424 	 */
425 	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
426 				&& current == policy->transition_task);
427 
428 wait:
429 	wait_event(policy->transition_wait, !policy->transition_ongoing);
430 
431 	spin_lock(&policy->transition_lock);
432 
433 	if (unlikely(policy->transition_ongoing)) {
434 		spin_unlock(&policy->transition_lock);
435 		goto wait;
436 	}
437 
438 	policy->transition_ongoing = true;
439 	policy->transition_task = current;
440 
441 	spin_unlock(&policy->transition_lock);
442 
443 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
444 }
445 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
446 
cpufreq_freq_transition_end(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)447 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
448 		struct cpufreq_freqs *freqs, int transition_failed)
449 {
450 	if (WARN_ON(!policy->transition_ongoing))
451 		return;
452 
453 	cpufreq_notify_post_transition(policy, freqs, transition_failed);
454 
455 	arch_set_freq_scale(policy->related_cpus,
456 			    policy->cur,
457 			    policy->cpuinfo.max_freq);
458 
459 	spin_lock(&policy->transition_lock);
460 	policy->transition_ongoing = false;
461 	policy->transition_task = NULL;
462 	spin_unlock(&policy->transition_lock);
463 
464 	wake_up(&policy->transition_wait);
465 }
466 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
467 
468 /*
469  * Fast frequency switching status count.  Positive means "enabled", negative
470  * means "disabled" and 0 means "not decided yet".
471  */
472 static int cpufreq_fast_switch_count;
473 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
474 
cpufreq_list_transition_notifiers(void)475 static void cpufreq_list_transition_notifiers(void)
476 {
477 	struct notifier_block *nb;
478 
479 	pr_info("Registered transition notifiers:\n");
480 
481 	mutex_lock(&cpufreq_transition_notifier_list.mutex);
482 
483 	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
484 		pr_info("%pS\n", nb->notifier_call);
485 
486 	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
487 }
488 
489 /**
490  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
491  * @policy: cpufreq policy to enable fast frequency switching for.
492  *
493  * Try to enable fast frequency switching for @policy.
494  *
495  * The attempt will fail if there is at least one transition notifier registered
496  * at this point, as fast frequency switching is quite fundamentally at odds
497  * with transition notifiers.  Thus if successful, it will make registration of
498  * transition notifiers fail going forward.
499  */
cpufreq_enable_fast_switch(struct cpufreq_policy * policy)500 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
501 {
502 	lockdep_assert_held(&policy->rwsem);
503 
504 	if (!policy->fast_switch_possible)
505 		return;
506 
507 	mutex_lock(&cpufreq_fast_switch_lock);
508 	if (cpufreq_fast_switch_count >= 0) {
509 		cpufreq_fast_switch_count++;
510 		policy->fast_switch_enabled = true;
511 	} else {
512 		pr_warn("CPU%u: Fast frequency switching not enabled\n",
513 			policy->cpu);
514 		cpufreq_list_transition_notifiers();
515 	}
516 	mutex_unlock(&cpufreq_fast_switch_lock);
517 }
518 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
519 
520 /**
521  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
522  * @policy: cpufreq policy to disable fast frequency switching for.
523  */
cpufreq_disable_fast_switch(struct cpufreq_policy * policy)524 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
525 {
526 	mutex_lock(&cpufreq_fast_switch_lock);
527 	if (policy->fast_switch_enabled) {
528 		policy->fast_switch_enabled = false;
529 		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
530 			cpufreq_fast_switch_count--;
531 	}
532 	mutex_unlock(&cpufreq_fast_switch_lock);
533 }
534 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
535 
__resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)536 static unsigned int __resolve_freq(struct cpufreq_policy *policy,
537 		unsigned int target_freq, unsigned int relation)
538 {
539 	unsigned int idx;
540 
541 	target_freq = clamp_val(target_freq, policy->min, policy->max);
542 
543 	if (!policy->freq_table)
544 		return target_freq;
545 
546 	idx = cpufreq_frequency_table_target(policy, target_freq, relation);
547 	policy->cached_resolved_idx = idx;
548 	policy->cached_target_freq = target_freq;
549 	return policy->freq_table[idx].frequency;
550 }
551 
552 /**
553  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
554  * one.
555  * @policy: associated policy to interrogate
556  * @target_freq: target frequency to resolve.
557  *
558  * The target to driver frequency mapping is cached in the policy.
559  *
560  * Return: Lowest driver-supported frequency greater than or equal to the
561  * given target_freq, subject to policy (min/max) and driver limitations.
562  */
cpufreq_driver_resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq)563 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
564 					 unsigned int target_freq)
565 {
566 	return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
567 }
568 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
569 
cpufreq_policy_transition_delay_us(struct cpufreq_policy * policy)570 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
571 {
572 	unsigned int latency;
573 
574 	if (policy->transition_delay_us)
575 		return policy->transition_delay_us;
576 
577 	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
578 	if (latency) {
579 		/*
580 		 * For platforms that can change the frequency very fast (< 10
581 		 * us), the above formula gives a decent transition delay. But
582 		 * for platforms where transition_latency is in milliseconds, it
583 		 * ends up giving unrealistic values.
584 		 *
585 		 * Cap the default transition delay to 10 ms, which seems to be
586 		 * a reasonable amount of time after which we should reevaluate
587 		 * the frequency.
588 		 */
589 		return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
590 	}
591 
592 	return LATENCY_MULTIPLIER;
593 }
594 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
595 
596 /*********************************************************************
597  *                          SYSFS INTERFACE                          *
598  *********************************************************************/
show_boost(struct kobject * kobj,struct kobj_attribute * attr,char * buf)599 static ssize_t show_boost(struct kobject *kobj,
600 			  struct kobj_attribute *attr, char *buf)
601 {
602 	return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
603 }
604 
store_boost(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)605 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
606 			   const char *buf, size_t count)
607 {
608 	int ret, enable;
609 
610 	ret = sscanf(buf, "%d", &enable);
611 	if (ret != 1 || enable < 0 || enable > 1)
612 		return -EINVAL;
613 
614 	if (cpufreq_boost_trigger_state(enable)) {
615 		pr_err("%s: Cannot %s BOOST!\n",
616 		       __func__, enable ? "enable" : "disable");
617 		return -EINVAL;
618 	}
619 
620 	pr_debug("%s: cpufreq BOOST %s\n",
621 		 __func__, enable ? "enabled" : "disabled");
622 
623 	return count;
624 }
625 define_one_global_rw(boost);
626 
show_local_boost(struct cpufreq_policy * policy,char * buf)627 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
628 {
629 	return sysfs_emit(buf, "%d\n", policy->boost_enabled);
630 }
631 
store_local_boost(struct cpufreq_policy * policy,const char * buf,size_t count)632 static ssize_t store_local_boost(struct cpufreq_policy *policy,
633 				 const char *buf, size_t count)
634 {
635 	int ret, enable;
636 
637 	ret = kstrtoint(buf, 10, &enable);
638 	if (ret || enable < 0 || enable > 1)
639 		return -EINVAL;
640 
641 	if (!cpufreq_driver->boost_enabled)
642 		return -EINVAL;
643 
644 	if (policy->boost_enabled == enable)
645 		return count;
646 
647 	cpus_read_lock();
648 	ret = cpufreq_driver->set_boost(policy, enable);
649 	cpus_read_unlock();
650 
651 	if (ret)
652 		return ret;
653 
654 	policy->boost_enabled = enable;
655 
656 	return count;
657 }
658 
659 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
660 
find_governor(const char * str_governor)661 static struct cpufreq_governor *find_governor(const char *str_governor)
662 {
663 	struct cpufreq_governor *t;
664 
665 	for_each_governor(t)
666 		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
667 			return t;
668 
669 	return NULL;
670 }
671 
get_governor(const char * str_governor)672 static struct cpufreq_governor *get_governor(const char *str_governor)
673 {
674 	struct cpufreq_governor *t;
675 
676 	mutex_lock(&cpufreq_governor_mutex);
677 	t = find_governor(str_governor);
678 	if (!t)
679 		goto unlock;
680 
681 	if (!try_module_get(t->owner))
682 		t = NULL;
683 
684 unlock:
685 	mutex_unlock(&cpufreq_governor_mutex);
686 
687 	return t;
688 }
689 
cpufreq_parse_policy(char * str_governor)690 static unsigned int cpufreq_parse_policy(char *str_governor)
691 {
692 	if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
693 		return CPUFREQ_POLICY_PERFORMANCE;
694 
695 	if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
696 		return CPUFREQ_POLICY_POWERSAVE;
697 
698 	return CPUFREQ_POLICY_UNKNOWN;
699 }
700 
701 /**
702  * cpufreq_parse_governor - parse a governor string only for has_target()
703  * @str_governor: Governor name.
704  */
cpufreq_parse_governor(char * str_governor)705 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
706 {
707 	struct cpufreq_governor *t;
708 
709 	t = get_governor(str_governor);
710 	if (t)
711 		return t;
712 
713 	if (request_module("cpufreq_%s", str_governor))
714 		return NULL;
715 
716 	return get_governor(str_governor);
717 }
718 
719 /*
720  * cpufreq_per_cpu_attr_read() / show_##file_name() -
721  * print out cpufreq information
722  *
723  * Write out information from cpufreq_driver->policy[cpu]; object must be
724  * "unsigned int".
725  */
726 
727 #define show_one(file_name, object)			\
728 static ssize_t show_##file_name				\
729 (struct cpufreq_policy *policy, char *buf)		\
730 {							\
731 	return sprintf(buf, "%u\n", policy->object);	\
732 }
733 
734 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
735 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
736 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
737 show_one(scaling_min_freq, min);
738 show_one(scaling_max_freq, max);
739 
arch_freq_get_on_cpu(int cpu)740 __weak unsigned int arch_freq_get_on_cpu(int cpu)
741 {
742 	return 0;
743 }
744 
show_scaling_cur_freq(struct cpufreq_policy * policy,char * buf)745 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
746 {
747 	ssize_t ret;
748 	unsigned int freq;
749 
750 	freq = arch_freq_get_on_cpu(policy->cpu);
751 	if (freq)
752 		ret = sprintf(buf, "%u\n", freq);
753 	else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
754 		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
755 	else
756 		ret = sprintf(buf, "%u\n", policy->cur);
757 	return ret;
758 }
759 
760 /*
761  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
762  */
763 #define store_one(file_name, object)			\
764 static ssize_t store_##file_name					\
765 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
766 {									\
767 	unsigned long val;						\
768 	int ret;							\
769 									\
770 	ret = kstrtoul(buf, 0, &val);					\
771 	if (ret)							\
772 		return ret;						\
773 									\
774 	ret = freq_qos_update_request(policy->object##_freq_req, val);\
775 	return ret >= 0 ? count : ret;					\
776 }
777 
778 store_one(scaling_min_freq, min);
779 store_one(scaling_max_freq, max);
780 
781 /*
782  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
783  */
show_cpuinfo_cur_freq(struct cpufreq_policy * policy,char * buf)784 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
785 					char *buf)
786 {
787 	unsigned int cur_freq = __cpufreq_get(policy);
788 
789 	if (cur_freq)
790 		return sprintf(buf, "%u\n", cur_freq);
791 
792 	return sprintf(buf, "<unknown>\n");
793 }
794 
795 /*
796  * show_scaling_governor - show the current policy for the specified CPU
797  */
show_scaling_governor(struct cpufreq_policy * policy,char * buf)798 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
799 {
800 	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
801 		return sprintf(buf, "powersave\n");
802 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
803 		return sprintf(buf, "performance\n");
804 	else if (policy->governor)
805 		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
806 				policy->governor->name);
807 	return -EINVAL;
808 }
809 
810 /*
811  * store_scaling_governor - store policy for the specified CPU
812  */
store_scaling_governor(struct cpufreq_policy * policy,const char * buf,size_t count)813 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
814 					const char *buf, size_t count)
815 {
816 	char str_governor[16];
817 	int ret;
818 
819 	ret = sscanf(buf, "%15s", str_governor);
820 	if (ret != 1)
821 		return -EINVAL;
822 
823 	if (cpufreq_driver->setpolicy) {
824 		unsigned int new_pol;
825 
826 		new_pol = cpufreq_parse_policy(str_governor);
827 		if (!new_pol)
828 			return -EINVAL;
829 
830 		ret = cpufreq_set_policy(policy, NULL, new_pol);
831 	} else {
832 		struct cpufreq_governor *new_gov;
833 
834 		new_gov = cpufreq_parse_governor(str_governor);
835 		if (!new_gov)
836 			return -EINVAL;
837 
838 		ret = cpufreq_set_policy(policy, new_gov,
839 					 CPUFREQ_POLICY_UNKNOWN);
840 
841 		module_put(new_gov->owner);
842 	}
843 
844 	return ret ? ret : count;
845 }
846 
847 /*
848  * show_scaling_driver - show the cpufreq driver currently loaded
849  */
show_scaling_driver(struct cpufreq_policy * policy,char * buf)850 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
851 {
852 	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
853 }
854 
855 /*
856  * show_scaling_available_governors - show the available CPUfreq governors
857  */
show_scaling_available_governors(struct cpufreq_policy * policy,char * buf)858 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
859 						char *buf)
860 {
861 	ssize_t i = 0;
862 	struct cpufreq_governor *t;
863 
864 	if (!has_target()) {
865 		i += sprintf(buf, "performance powersave");
866 		goto out;
867 	}
868 
869 	mutex_lock(&cpufreq_governor_mutex);
870 	for_each_governor(t) {
871 		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
872 		    - (CPUFREQ_NAME_LEN + 2)))
873 			break;
874 		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
875 	}
876 	mutex_unlock(&cpufreq_governor_mutex);
877 out:
878 	i += sprintf(&buf[i], "\n");
879 	return i;
880 }
881 
cpufreq_show_cpus(const struct cpumask * mask,char * buf)882 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
883 {
884 	ssize_t i = 0;
885 	unsigned int cpu;
886 
887 	for_each_cpu(cpu, mask) {
888 		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u ", cpu);
889 		if (i >= (PAGE_SIZE - 5))
890 			break;
891 	}
892 
893 	/* Remove the extra space at the end */
894 	i--;
895 
896 	i += sprintf(&buf[i], "\n");
897 	return i;
898 }
899 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
900 
901 /*
902  * show_related_cpus - show the CPUs affected by each transition even if
903  * hw coordination is in use
904  */
show_related_cpus(struct cpufreq_policy * policy,char * buf)905 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
906 {
907 	return cpufreq_show_cpus(policy->related_cpus, buf);
908 }
909 
910 /*
911  * show_affected_cpus - show the CPUs affected by each transition
912  */
show_affected_cpus(struct cpufreq_policy * policy,char * buf)913 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
914 {
915 	return cpufreq_show_cpus(policy->cpus, buf);
916 }
917 
store_scaling_setspeed(struct cpufreq_policy * policy,const char * buf,size_t count)918 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
919 					const char *buf, size_t count)
920 {
921 	unsigned int freq = 0;
922 	unsigned int ret;
923 
924 	if (!policy->governor || !policy->governor->store_setspeed)
925 		return -EINVAL;
926 
927 	ret = sscanf(buf, "%u", &freq);
928 	if (ret != 1)
929 		return -EINVAL;
930 
931 	policy->governor->store_setspeed(policy, freq);
932 
933 	return count;
934 }
935 
show_scaling_setspeed(struct cpufreq_policy * policy,char * buf)936 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
937 {
938 	if (!policy->governor || !policy->governor->show_setspeed)
939 		return sprintf(buf, "<unsupported>\n");
940 
941 	return policy->governor->show_setspeed(policy, buf);
942 }
943 
944 /*
945  * show_bios_limit - show the current cpufreq HW/BIOS limitation
946  */
show_bios_limit(struct cpufreq_policy * policy,char * buf)947 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
948 {
949 	unsigned int limit;
950 	int ret;
951 	ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
952 	if (!ret)
953 		return sprintf(buf, "%u\n", limit);
954 	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
955 }
956 
957 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
958 cpufreq_freq_attr_ro(cpuinfo_min_freq);
959 cpufreq_freq_attr_ro(cpuinfo_max_freq);
960 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
961 cpufreq_freq_attr_ro(scaling_available_governors);
962 cpufreq_freq_attr_ro(scaling_driver);
963 cpufreq_freq_attr_ro(scaling_cur_freq);
964 cpufreq_freq_attr_ro(bios_limit);
965 cpufreq_freq_attr_ro(related_cpus);
966 cpufreq_freq_attr_ro(affected_cpus);
967 cpufreq_freq_attr_rw(scaling_min_freq);
968 cpufreq_freq_attr_rw(scaling_max_freq);
969 cpufreq_freq_attr_rw(scaling_governor);
970 cpufreq_freq_attr_rw(scaling_setspeed);
971 
972 static struct attribute *cpufreq_attrs[] = {
973 	&cpuinfo_min_freq.attr,
974 	&cpuinfo_max_freq.attr,
975 	&cpuinfo_transition_latency.attr,
976 	&scaling_min_freq.attr,
977 	&scaling_max_freq.attr,
978 	&affected_cpus.attr,
979 	&related_cpus.attr,
980 	&scaling_governor.attr,
981 	&scaling_driver.attr,
982 	&scaling_available_governors.attr,
983 	&scaling_setspeed.attr,
984 	NULL
985 };
986 ATTRIBUTE_GROUPS(cpufreq);
987 
988 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
989 #define to_attr(a) container_of(a, struct freq_attr, attr)
990 
show(struct kobject * kobj,struct attribute * attr,char * buf)991 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
992 {
993 	struct cpufreq_policy *policy = to_policy(kobj);
994 	struct freq_attr *fattr = to_attr(attr);
995 	ssize_t ret = -EBUSY;
996 
997 	if (!fattr->show)
998 		return -EIO;
999 
1000 	down_read(&policy->rwsem);
1001 	if (likely(!policy_is_inactive(policy)))
1002 		ret = fattr->show(policy, buf);
1003 	up_read(&policy->rwsem);
1004 
1005 	return ret;
1006 }
1007 
store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)1008 static ssize_t store(struct kobject *kobj, struct attribute *attr,
1009 		     const char *buf, size_t count)
1010 {
1011 	struct cpufreq_policy *policy = to_policy(kobj);
1012 	struct freq_attr *fattr = to_attr(attr);
1013 	ssize_t ret = -EBUSY;
1014 
1015 	if (!fattr->store)
1016 		return -EIO;
1017 
1018 	down_write(&policy->rwsem);
1019 	if (likely(!policy_is_inactive(policy)))
1020 		ret = fattr->store(policy, buf, count);
1021 	up_write(&policy->rwsem);
1022 
1023 	return ret;
1024 }
1025 
cpufreq_sysfs_release(struct kobject * kobj)1026 static void cpufreq_sysfs_release(struct kobject *kobj)
1027 {
1028 	struct cpufreq_policy *policy = to_policy(kobj);
1029 	pr_debug("last reference is dropped\n");
1030 	complete(&policy->kobj_unregister);
1031 }
1032 
1033 static const struct sysfs_ops sysfs_ops = {
1034 	.show	= show,
1035 	.store	= store,
1036 };
1037 
1038 static const struct kobj_type ktype_cpufreq = {
1039 	.sysfs_ops	= &sysfs_ops,
1040 	.default_groups	= cpufreq_groups,
1041 	.release	= cpufreq_sysfs_release,
1042 };
1043 
add_cpu_dev_symlink(struct cpufreq_policy * policy,unsigned int cpu,struct device * dev)1044 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1045 				struct device *dev)
1046 {
1047 	if (unlikely(!dev))
1048 		return;
1049 
1050 	if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1051 		return;
1052 
1053 	dev_dbg(dev, "%s: Adding symlink\n", __func__);
1054 	if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1055 		dev_err(dev, "cpufreq symlink creation failed\n");
1056 }
1057 
remove_cpu_dev_symlink(struct cpufreq_policy * policy,int cpu,struct device * dev)1058 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1059 				   struct device *dev)
1060 {
1061 	dev_dbg(dev, "%s: Removing symlink\n", __func__);
1062 	sysfs_remove_link(&dev->kobj, "cpufreq");
1063 	cpumask_clear_cpu(cpu, policy->real_cpus);
1064 }
1065 
cpufreq_add_dev_interface(struct cpufreq_policy * policy)1066 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1067 {
1068 	struct freq_attr **drv_attr;
1069 	int ret = 0;
1070 
1071 	/* set up files for this cpu device */
1072 	drv_attr = cpufreq_driver->attr;
1073 	while (drv_attr && *drv_attr) {
1074 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1075 		if (ret)
1076 			return ret;
1077 		drv_attr++;
1078 	}
1079 	if (cpufreq_driver->get) {
1080 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1081 		if (ret)
1082 			return ret;
1083 	}
1084 
1085 	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1086 	if (ret)
1087 		return ret;
1088 
1089 	if (cpufreq_driver->bios_limit) {
1090 		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1091 		if (ret)
1092 			return ret;
1093 	}
1094 
1095 	if (cpufreq_boost_supported()) {
1096 		ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
1097 		if (ret)
1098 			return ret;
1099 	}
1100 
1101 	return 0;
1102 }
1103 
cpufreq_init_policy(struct cpufreq_policy * policy)1104 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1105 {
1106 	struct cpufreq_governor *gov = NULL;
1107 	unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1108 	int ret;
1109 
1110 	if (has_target()) {
1111 		/* Update policy governor to the one used before hotplug. */
1112 		gov = get_governor(policy->last_governor);
1113 		if (gov) {
1114 			pr_debug("Restoring governor %s for cpu %d\n",
1115 				 gov->name, policy->cpu);
1116 		} else {
1117 			gov = get_governor(default_governor);
1118 		}
1119 
1120 		if (!gov) {
1121 			gov = cpufreq_default_governor();
1122 			__module_get(gov->owner);
1123 		}
1124 
1125 	} else {
1126 
1127 		/* Use the default policy if there is no last_policy. */
1128 		if (policy->last_policy) {
1129 			pol = policy->last_policy;
1130 		} else {
1131 			pol = cpufreq_parse_policy(default_governor);
1132 			/*
1133 			 * In case the default governor is neither "performance"
1134 			 * nor "powersave", fall back to the initial policy
1135 			 * value set by the driver.
1136 			 */
1137 			if (pol == CPUFREQ_POLICY_UNKNOWN)
1138 				pol = policy->policy;
1139 		}
1140 		if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1141 		    pol != CPUFREQ_POLICY_POWERSAVE)
1142 			return -ENODATA;
1143 	}
1144 
1145 	ret = cpufreq_set_policy(policy, gov, pol);
1146 	if (gov)
1147 		module_put(gov->owner);
1148 
1149 	return ret;
1150 }
1151 
cpufreq_add_policy_cpu(struct cpufreq_policy * policy,unsigned int cpu)1152 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1153 {
1154 	int ret = 0;
1155 
1156 	/* Has this CPU been taken care of already? */
1157 	if (cpumask_test_cpu(cpu, policy->cpus))
1158 		return 0;
1159 
1160 	down_write(&policy->rwsem);
1161 	if (has_target())
1162 		cpufreq_stop_governor(policy);
1163 
1164 	cpumask_set_cpu(cpu, policy->cpus);
1165 
1166 	if (has_target()) {
1167 		ret = cpufreq_start_governor(policy);
1168 		if (ret)
1169 			pr_err("%s: Failed to start governor\n", __func__);
1170 	}
1171 	up_write(&policy->rwsem);
1172 	return ret;
1173 }
1174 
refresh_frequency_limits(struct cpufreq_policy * policy)1175 void refresh_frequency_limits(struct cpufreq_policy *policy)
1176 {
1177 	if (!policy_is_inactive(policy)) {
1178 		pr_debug("updating policy for CPU %u\n", policy->cpu);
1179 
1180 		cpufreq_set_policy(policy, policy->governor, policy->policy);
1181 	}
1182 }
1183 EXPORT_SYMBOL(refresh_frequency_limits);
1184 
handle_update(struct work_struct * work)1185 static void handle_update(struct work_struct *work)
1186 {
1187 	struct cpufreq_policy *policy =
1188 		container_of(work, struct cpufreq_policy, update);
1189 
1190 	pr_debug("handle_update for cpu %u called\n", policy->cpu);
1191 	down_write(&policy->rwsem);
1192 	refresh_frequency_limits(policy);
1193 	up_write(&policy->rwsem);
1194 }
1195 
cpufreq_notifier_min(struct notifier_block * nb,unsigned long freq,void * data)1196 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1197 				void *data)
1198 {
1199 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1200 
1201 	schedule_work(&policy->update);
1202 	return 0;
1203 }
1204 
cpufreq_notifier_max(struct notifier_block * nb,unsigned long freq,void * data)1205 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1206 				void *data)
1207 {
1208 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1209 
1210 	schedule_work(&policy->update);
1211 	return 0;
1212 }
1213 
cpufreq_policy_put_kobj(struct cpufreq_policy * policy)1214 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1215 {
1216 	struct kobject *kobj;
1217 	struct completion *cmp;
1218 
1219 	down_write(&policy->rwsem);
1220 	cpufreq_stats_free_table(policy);
1221 	kobj = &policy->kobj;
1222 	cmp = &policy->kobj_unregister;
1223 	up_write(&policy->rwsem);
1224 	kobject_put(kobj);
1225 
1226 	/*
1227 	 * We need to make sure that the underlying kobj is
1228 	 * actually not referenced anymore by anybody before we
1229 	 * proceed with unloading.
1230 	 */
1231 	pr_debug("waiting for dropping of refcount\n");
1232 	wait_for_completion(cmp);
1233 	pr_debug("wait complete\n");
1234 }
1235 
cpufreq_policy_alloc(unsigned int cpu)1236 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1237 {
1238 	struct cpufreq_policy *policy;
1239 	struct device *dev = get_cpu_device(cpu);
1240 	int ret;
1241 
1242 	if (!dev)
1243 		return NULL;
1244 
1245 	policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1246 	if (!policy)
1247 		return NULL;
1248 
1249 	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1250 		goto err_free_policy;
1251 
1252 	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1253 		goto err_free_cpumask;
1254 
1255 	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1256 		goto err_free_rcpumask;
1257 
1258 	init_completion(&policy->kobj_unregister);
1259 	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1260 				   cpufreq_global_kobject, "policy%u", cpu);
1261 	if (ret) {
1262 		dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1263 		/*
1264 		 * The entire policy object will be freed below, but the extra
1265 		 * memory allocated for the kobject name needs to be freed by
1266 		 * releasing the kobject.
1267 		 */
1268 		kobject_put(&policy->kobj);
1269 		goto err_free_real_cpus;
1270 	}
1271 
1272 	freq_constraints_init(&policy->constraints);
1273 
1274 	policy->nb_min.notifier_call = cpufreq_notifier_min;
1275 	policy->nb_max.notifier_call = cpufreq_notifier_max;
1276 
1277 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1278 				    &policy->nb_min);
1279 	if (ret) {
1280 		dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1281 			ret, cpu);
1282 		goto err_kobj_remove;
1283 	}
1284 
1285 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1286 				    &policy->nb_max);
1287 	if (ret) {
1288 		dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1289 			ret, cpu);
1290 		goto err_min_qos_notifier;
1291 	}
1292 
1293 	INIT_LIST_HEAD(&policy->policy_list);
1294 	init_rwsem(&policy->rwsem);
1295 	spin_lock_init(&policy->transition_lock);
1296 	init_waitqueue_head(&policy->transition_wait);
1297 	INIT_WORK(&policy->update, handle_update);
1298 
1299 	policy->cpu = cpu;
1300 	return policy;
1301 
1302 err_min_qos_notifier:
1303 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1304 				 &policy->nb_min);
1305 err_kobj_remove:
1306 	cpufreq_policy_put_kobj(policy);
1307 err_free_real_cpus:
1308 	free_cpumask_var(policy->real_cpus);
1309 err_free_rcpumask:
1310 	free_cpumask_var(policy->related_cpus);
1311 err_free_cpumask:
1312 	free_cpumask_var(policy->cpus);
1313 err_free_policy:
1314 	kfree(policy);
1315 
1316 	return NULL;
1317 }
1318 
cpufreq_policy_free(struct cpufreq_policy * policy)1319 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1320 {
1321 	unsigned long flags;
1322 	int cpu;
1323 
1324 	/*
1325 	 * The callers must ensure the policy is inactive by now, to avoid any
1326 	 * races with show()/store() callbacks.
1327 	 */
1328 	if (unlikely(!policy_is_inactive(policy)))
1329 		pr_warn("%s: Freeing active policy\n", __func__);
1330 
1331 	/* Remove policy from list */
1332 	write_lock_irqsave(&cpufreq_driver_lock, flags);
1333 	list_del(&policy->policy_list);
1334 
1335 	for_each_cpu(cpu, policy->related_cpus)
1336 		per_cpu(cpufreq_cpu_data, cpu) = NULL;
1337 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1338 
1339 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1340 				 &policy->nb_max);
1341 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1342 				 &policy->nb_min);
1343 
1344 	/* Cancel any pending policy->update work before freeing the policy. */
1345 	cancel_work_sync(&policy->update);
1346 
1347 	if (policy->max_freq_req) {
1348 		/*
1349 		 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1350 		 * notification, since CPUFREQ_CREATE_POLICY notification was
1351 		 * sent after adding max_freq_req earlier.
1352 		 */
1353 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1354 					     CPUFREQ_REMOVE_POLICY, policy);
1355 		freq_qos_remove_request(policy->max_freq_req);
1356 	}
1357 
1358 	freq_qos_remove_request(policy->min_freq_req);
1359 	kfree(policy->min_freq_req);
1360 
1361 	cpufreq_policy_put_kobj(policy);
1362 	free_cpumask_var(policy->real_cpus);
1363 	free_cpumask_var(policy->related_cpus);
1364 	free_cpumask_var(policy->cpus);
1365 	kfree(policy);
1366 }
1367 
cpufreq_online(unsigned int cpu)1368 static int cpufreq_online(unsigned int cpu)
1369 {
1370 	struct cpufreq_policy *policy;
1371 	bool new_policy;
1372 	unsigned long flags;
1373 	unsigned int j;
1374 	int ret;
1375 
1376 	pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1377 
1378 	/* Check if this CPU already has a policy to manage it */
1379 	policy = per_cpu(cpufreq_cpu_data, cpu);
1380 	if (policy) {
1381 		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1382 		if (!policy_is_inactive(policy))
1383 			return cpufreq_add_policy_cpu(policy, cpu);
1384 
1385 		/* This is the only online CPU for the policy.  Start over. */
1386 		new_policy = false;
1387 		down_write(&policy->rwsem);
1388 		policy->cpu = cpu;
1389 		policy->governor = NULL;
1390 	} else {
1391 		new_policy = true;
1392 		policy = cpufreq_policy_alloc(cpu);
1393 		if (!policy)
1394 			return -ENOMEM;
1395 		down_write(&policy->rwsem);
1396 	}
1397 
1398 	if (!new_policy && cpufreq_driver->online) {
1399 		/* Recover policy->cpus using related_cpus */
1400 		cpumask_copy(policy->cpus, policy->related_cpus);
1401 
1402 		ret = cpufreq_driver->online(policy);
1403 		if (ret) {
1404 			pr_debug("%s: %d: initialization failed\n", __func__,
1405 				 __LINE__);
1406 			goto out_exit_policy;
1407 		}
1408 	} else {
1409 		cpumask_copy(policy->cpus, cpumask_of(cpu));
1410 
1411 		/*
1412 		 * Call driver. From then on the cpufreq must be able
1413 		 * to accept all calls to ->verify and ->setpolicy for this CPU.
1414 		 */
1415 		ret = cpufreq_driver->init(policy);
1416 		if (ret) {
1417 			pr_debug("%s: %d: initialization failed\n", __func__,
1418 				 __LINE__);
1419 			goto out_free_policy;
1420 		}
1421 
1422 		/*
1423 		 * The initialization has succeeded and the policy is online.
1424 		 * If there is a problem with its frequency table, take it
1425 		 * offline and drop it.
1426 		 */
1427 		ret = cpufreq_table_validate_and_sort(policy);
1428 		if (ret)
1429 			goto out_offline_policy;
1430 
1431 		/* related_cpus should at least include policy->cpus. */
1432 		cpumask_copy(policy->related_cpus, policy->cpus);
1433 	}
1434 
1435 	/*
1436 	 * affected cpus must always be the one, which are online. We aren't
1437 	 * managing offline cpus here.
1438 	 */
1439 	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1440 
1441 	if (new_policy) {
1442 		for_each_cpu(j, policy->related_cpus) {
1443 			per_cpu(cpufreq_cpu_data, j) = policy;
1444 			add_cpu_dev_symlink(policy, j, get_cpu_device(j));
1445 		}
1446 
1447 		policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1448 					       GFP_KERNEL);
1449 		if (!policy->min_freq_req) {
1450 			ret = -ENOMEM;
1451 			goto out_destroy_policy;
1452 		}
1453 
1454 		ret = freq_qos_add_request(&policy->constraints,
1455 					   policy->min_freq_req, FREQ_QOS_MIN,
1456 					   FREQ_QOS_MIN_DEFAULT_VALUE);
1457 		if (ret < 0) {
1458 			/*
1459 			 * So we don't call freq_qos_remove_request() for an
1460 			 * uninitialized request.
1461 			 */
1462 			kfree(policy->min_freq_req);
1463 			policy->min_freq_req = NULL;
1464 			goto out_destroy_policy;
1465 		}
1466 
1467 		/*
1468 		 * This must be initialized right here to avoid calling
1469 		 * freq_qos_remove_request() on uninitialized request in case
1470 		 * of errors.
1471 		 */
1472 		policy->max_freq_req = policy->min_freq_req + 1;
1473 
1474 		ret = freq_qos_add_request(&policy->constraints,
1475 					   policy->max_freq_req, FREQ_QOS_MAX,
1476 					   FREQ_QOS_MAX_DEFAULT_VALUE);
1477 		if (ret < 0) {
1478 			policy->max_freq_req = NULL;
1479 			goto out_destroy_policy;
1480 		}
1481 
1482 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1483 				CPUFREQ_CREATE_POLICY, policy);
1484 	}
1485 
1486 	if (cpufreq_driver->get && has_target()) {
1487 		policy->cur = cpufreq_driver->get(policy->cpu);
1488 		if (!policy->cur) {
1489 			ret = -EIO;
1490 			pr_err("%s: ->get() failed\n", __func__);
1491 			goto out_destroy_policy;
1492 		}
1493 	}
1494 
1495 	/*
1496 	 * Sometimes boot loaders set CPU frequency to a value outside of
1497 	 * frequency table present with cpufreq core. In such cases CPU might be
1498 	 * unstable if it has to run on that frequency for long duration of time
1499 	 * and so its better to set it to a frequency which is specified in
1500 	 * freq-table. This also makes cpufreq stats inconsistent as
1501 	 * cpufreq-stats would fail to register because current frequency of CPU
1502 	 * isn't found in freq-table.
1503 	 *
1504 	 * Because we don't want this change to effect boot process badly, we go
1505 	 * for the next freq which is >= policy->cur ('cur' must be set by now,
1506 	 * otherwise we will end up setting freq to lowest of the table as 'cur'
1507 	 * is initialized to zero).
1508 	 *
1509 	 * We are passing target-freq as "policy->cur - 1" otherwise
1510 	 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1511 	 * equal to target-freq.
1512 	 */
1513 	if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1514 	    && has_target()) {
1515 		unsigned int old_freq = policy->cur;
1516 
1517 		/* Are we running at unknown frequency ? */
1518 		ret = cpufreq_frequency_table_get_index(policy, old_freq);
1519 		if (ret == -EINVAL) {
1520 			ret = __cpufreq_driver_target(policy, old_freq - 1,
1521 						      CPUFREQ_RELATION_L);
1522 
1523 			/*
1524 			 * Reaching here after boot in a few seconds may not
1525 			 * mean that system will remain stable at "unknown"
1526 			 * frequency for longer duration. Hence, a BUG_ON().
1527 			 */
1528 			BUG_ON(ret);
1529 			pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
1530 				__func__, policy->cpu, old_freq, policy->cur);
1531 		}
1532 	}
1533 
1534 	if (new_policy) {
1535 		ret = cpufreq_add_dev_interface(policy);
1536 		if (ret)
1537 			goto out_destroy_policy;
1538 
1539 		cpufreq_stats_create_table(policy);
1540 
1541 		write_lock_irqsave(&cpufreq_driver_lock, flags);
1542 		list_add(&policy->policy_list, &cpufreq_policy_list);
1543 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1544 
1545 		/*
1546 		 * Register with the energy model before
1547 		 * sched_cpufreq_governor_change() is called, which will result
1548 		 * in rebuilding of the sched domains, which should only be done
1549 		 * once the energy model is properly initialized for the policy
1550 		 * first.
1551 		 *
1552 		 * Also, this should be called before the policy is registered
1553 		 * with cooling framework.
1554 		 */
1555 		if (cpufreq_driver->register_em)
1556 			cpufreq_driver->register_em(policy);
1557 	}
1558 
1559 	ret = cpufreq_init_policy(policy);
1560 	if (ret) {
1561 		pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1562 		       __func__, cpu, ret);
1563 		goto out_destroy_policy;
1564 	}
1565 
1566 	up_write(&policy->rwsem);
1567 
1568 	kobject_uevent(&policy->kobj, KOBJ_ADD);
1569 
1570 	/* Callback for handling stuff after policy is ready */
1571 	if (cpufreq_driver->ready)
1572 		cpufreq_driver->ready(policy);
1573 
1574 	if (cpufreq_thermal_control_enabled(cpufreq_driver))
1575 		policy->cdev = of_cpufreq_cooling_register(policy);
1576 
1577 	pr_debug("initialization complete\n");
1578 
1579 	return 0;
1580 
1581 out_destroy_policy:
1582 	for_each_cpu(j, policy->real_cpus)
1583 		remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
1584 
1585 out_offline_policy:
1586 	if (cpufreq_driver->offline)
1587 		cpufreq_driver->offline(policy);
1588 
1589 out_exit_policy:
1590 	if (cpufreq_driver->exit)
1591 		cpufreq_driver->exit(policy);
1592 
1593 out_free_policy:
1594 	cpumask_clear(policy->cpus);
1595 	up_write(&policy->rwsem);
1596 
1597 	cpufreq_policy_free(policy);
1598 	return ret;
1599 }
1600 
1601 /**
1602  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1603  * @dev: CPU device.
1604  * @sif: Subsystem interface structure pointer (not used)
1605  */
cpufreq_add_dev(struct device * dev,struct subsys_interface * sif)1606 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1607 {
1608 	struct cpufreq_policy *policy;
1609 	unsigned cpu = dev->id;
1610 	int ret;
1611 
1612 	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1613 
1614 	if (cpu_online(cpu)) {
1615 		ret = cpufreq_online(cpu);
1616 		if (ret)
1617 			return ret;
1618 	}
1619 
1620 	/* Create sysfs link on CPU registration */
1621 	policy = per_cpu(cpufreq_cpu_data, cpu);
1622 	if (policy)
1623 		add_cpu_dev_symlink(policy, cpu, dev);
1624 
1625 	return 0;
1626 }
1627 
__cpufreq_offline(unsigned int cpu,struct cpufreq_policy * policy)1628 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1629 {
1630 	int ret;
1631 
1632 	if (has_target())
1633 		cpufreq_stop_governor(policy);
1634 
1635 	cpumask_clear_cpu(cpu, policy->cpus);
1636 
1637 	if (!policy_is_inactive(policy)) {
1638 		/* Nominate a new CPU if necessary. */
1639 		if (cpu == policy->cpu)
1640 			policy->cpu = cpumask_any(policy->cpus);
1641 
1642 		/* Start the governor again for the active policy. */
1643 		if (has_target()) {
1644 			ret = cpufreq_start_governor(policy);
1645 			if (ret)
1646 				pr_err("%s: Failed to start governor\n", __func__);
1647 		}
1648 
1649 		return;
1650 	}
1651 
1652 	if (has_target())
1653 		strncpy(policy->last_governor, policy->governor->name,
1654 			CPUFREQ_NAME_LEN);
1655 	else
1656 		policy->last_policy = policy->policy;
1657 
1658 	if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1659 		cpufreq_cooling_unregister(policy->cdev);
1660 		policy->cdev = NULL;
1661 	}
1662 
1663 	if (has_target())
1664 		cpufreq_exit_governor(policy);
1665 
1666 	/*
1667 	 * Perform the ->offline() during light-weight tear-down, as
1668 	 * that allows fast recovery when the CPU comes back.
1669 	 */
1670 	if (cpufreq_driver->offline) {
1671 		cpufreq_driver->offline(policy);
1672 	} else if (cpufreq_driver->exit) {
1673 		cpufreq_driver->exit(policy);
1674 		policy->freq_table = NULL;
1675 	}
1676 }
1677 
cpufreq_offline(unsigned int cpu)1678 static int cpufreq_offline(unsigned int cpu)
1679 {
1680 	struct cpufreq_policy *policy;
1681 
1682 	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1683 
1684 	policy = cpufreq_cpu_get_raw(cpu);
1685 	if (!policy) {
1686 		pr_debug("%s: No cpu_data found\n", __func__);
1687 		return 0;
1688 	}
1689 
1690 	down_write(&policy->rwsem);
1691 
1692 	__cpufreq_offline(cpu, policy);
1693 
1694 	up_write(&policy->rwsem);
1695 	return 0;
1696 }
1697 
1698 /*
1699  * cpufreq_remove_dev - remove a CPU device
1700  *
1701  * Removes the cpufreq interface for a CPU device.
1702  */
cpufreq_remove_dev(struct device * dev,struct subsys_interface * sif)1703 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1704 {
1705 	unsigned int cpu = dev->id;
1706 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1707 
1708 	if (!policy)
1709 		return;
1710 
1711 	down_write(&policy->rwsem);
1712 
1713 	if (cpu_online(cpu))
1714 		__cpufreq_offline(cpu, policy);
1715 
1716 	remove_cpu_dev_symlink(policy, cpu, dev);
1717 
1718 	if (!cpumask_empty(policy->real_cpus)) {
1719 		up_write(&policy->rwsem);
1720 		return;
1721 	}
1722 
1723 	/* We did light-weight exit earlier, do full tear down now */
1724 	if (cpufreq_driver->offline)
1725 		cpufreq_driver->exit(policy);
1726 
1727 	up_write(&policy->rwsem);
1728 
1729 	cpufreq_policy_free(policy);
1730 }
1731 
1732 /**
1733  * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1734  * @policy: Policy managing CPUs.
1735  * @new_freq: New CPU frequency.
1736  *
1737  * Adjust to the current frequency first and clean up later by either calling
1738  * cpufreq_update_policy(), or scheduling handle_update().
1739  */
cpufreq_out_of_sync(struct cpufreq_policy * policy,unsigned int new_freq)1740 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1741 				unsigned int new_freq)
1742 {
1743 	struct cpufreq_freqs freqs;
1744 
1745 	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1746 		 policy->cur, new_freq);
1747 
1748 	freqs.old = policy->cur;
1749 	freqs.new = new_freq;
1750 
1751 	cpufreq_freq_transition_begin(policy, &freqs);
1752 	cpufreq_freq_transition_end(policy, &freqs, 0);
1753 }
1754 
cpufreq_verify_current_freq(struct cpufreq_policy * policy,bool update)1755 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1756 {
1757 	unsigned int new_freq;
1758 
1759 	new_freq = cpufreq_driver->get(policy->cpu);
1760 	if (!new_freq)
1761 		return 0;
1762 
1763 	/*
1764 	 * If fast frequency switching is used with the given policy, the check
1765 	 * against policy->cur is pointless, so skip it in that case.
1766 	 */
1767 	if (policy->fast_switch_enabled || !has_target())
1768 		return new_freq;
1769 
1770 	if (policy->cur != new_freq) {
1771 		/*
1772 		 * For some platforms, the frequency returned by hardware may be
1773 		 * slightly different from what is provided in the frequency
1774 		 * table, for example hardware may return 499 MHz instead of 500
1775 		 * MHz. In such cases it is better to avoid getting into
1776 		 * unnecessary frequency updates.
1777 		 */
1778 		if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1779 			return policy->cur;
1780 
1781 		cpufreq_out_of_sync(policy, new_freq);
1782 		if (update)
1783 			schedule_work(&policy->update);
1784 	}
1785 
1786 	return new_freq;
1787 }
1788 
1789 /**
1790  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1791  * @cpu: CPU number
1792  *
1793  * This is the last known freq, without actually getting it from the driver.
1794  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1795  */
cpufreq_quick_get(unsigned int cpu)1796 unsigned int cpufreq_quick_get(unsigned int cpu)
1797 {
1798 	struct cpufreq_policy *policy;
1799 	unsigned int ret_freq = 0;
1800 	unsigned long flags;
1801 
1802 	read_lock_irqsave(&cpufreq_driver_lock, flags);
1803 
1804 	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1805 		ret_freq = cpufreq_driver->get(cpu);
1806 		read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1807 		return ret_freq;
1808 	}
1809 
1810 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1811 
1812 	policy = cpufreq_cpu_get(cpu);
1813 	if (policy) {
1814 		ret_freq = policy->cur;
1815 		cpufreq_cpu_put(policy);
1816 	}
1817 
1818 	return ret_freq;
1819 }
1820 EXPORT_SYMBOL(cpufreq_quick_get);
1821 
1822 /**
1823  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1824  * @cpu: CPU number
1825  *
1826  * Just return the max possible frequency for a given CPU.
1827  */
cpufreq_quick_get_max(unsigned int cpu)1828 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1829 {
1830 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1831 	unsigned int ret_freq = 0;
1832 
1833 	if (policy) {
1834 		ret_freq = policy->max;
1835 		cpufreq_cpu_put(policy);
1836 	}
1837 
1838 	return ret_freq;
1839 }
1840 EXPORT_SYMBOL(cpufreq_quick_get_max);
1841 
1842 /**
1843  * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1844  * @cpu: CPU number
1845  *
1846  * The default return value is the max_freq field of cpuinfo.
1847  */
cpufreq_get_hw_max_freq(unsigned int cpu)1848 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1849 {
1850 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1851 	unsigned int ret_freq = 0;
1852 
1853 	if (policy) {
1854 		ret_freq = policy->cpuinfo.max_freq;
1855 		cpufreq_cpu_put(policy);
1856 	}
1857 
1858 	return ret_freq;
1859 }
1860 EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1861 
__cpufreq_get(struct cpufreq_policy * policy)1862 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1863 {
1864 	if (unlikely(policy_is_inactive(policy)))
1865 		return 0;
1866 
1867 	return cpufreq_verify_current_freq(policy, true);
1868 }
1869 
1870 /**
1871  * cpufreq_get - get the current CPU frequency (in kHz)
1872  * @cpu: CPU number
1873  *
1874  * Get the CPU current (static) CPU frequency
1875  */
cpufreq_get(unsigned int cpu)1876 unsigned int cpufreq_get(unsigned int cpu)
1877 {
1878 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1879 	unsigned int ret_freq = 0;
1880 
1881 	if (policy) {
1882 		down_read(&policy->rwsem);
1883 		if (cpufreq_driver->get)
1884 			ret_freq = __cpufreq_get(policy);
1885 		up_read(&policy->rwsem);
1886 
1887 		cpufreq_cpu_put(policy);
1888 	}
1889 
1890 	return ret_freq;
1891 }
1892 EXPORT_SYMBOL(cpufreq_get);
1893 
1894 static struct subsys_interface cpufreq_interface = {
1895 	.name		= "cpufreq",
1896 	.subsys		= &cpu_subsys,
1897 	.add_dev	= cpufreq_add_dev,
1898 	.remove_dev	= cpufreq_remove_dev,
1899 };
1900 
1901 /*
1902  * In case platform wants some specific frequency to be configured
1903  * during suspend..
1904  */
cpufreq_generic_suspend(struct cpufreq_policy * policy)1905 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1906 {
1907 	int ret;
1908 
1909 	if (!policy->suspend_freq) {
1910 		pr_debug("%s: suspend_freq not defined\n", __func__);
1911 		return 0;
1912 	}
1913 
1914 	pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1915 			policy->suspend_freq);
1916 
1917 	ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1918 			CPUFREQ_RELATION_H);
1919 	if (ret)
1920 		pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1921 				__func__, policy->suspend_freq, ret);
1922 
1923 	return ret;
1924 }
1925 EXPORT_SYMBOL(cpufreq_generic_suspend);
1926 
1927 /**
1928  * cpufreq_suspend() - Suspend CPUFreq governors.
1929  *
1930  * Called during system wide Suspend/Hibernate cycles for suspending governors
1931  * as some platforms can't change frequency after this point in suspend cycle.
1932  * Because some of the devices (like: i2c, regulators, etc) they use for
1933  * changing frequency are suspended quickly after this point.
1934  */
cpufreq_suspend(void)1935 void cpufreq_suspend(void)
1936 {
1937 	struct cpufreq_policy *policy;
1938 
1939 	if (!cpufreq_driver)
1940 		return;
1941 
1942 	if (!has_target() && !cpufreq_driver->suspend)
1943 		goto suspend;
1944 
1945 	pr_debug("%s: Suspending Governors\n", __func__);
1946 
1947 	for_each_active_policy(policy) {
1948 		if (has_target()) {
1949 			down_write(&policy->rwsem);
1950 			cpufreq_stop_governor(policy);
1951 			up_write(&policy->rwsem);
1952 		}
1953 
1954 		if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1955 			pr_err("%s: Failed to suspend driver: %s\n", __func__,
1956 				cpufreq_driver->name);
1957 	}
1958 
1959 suspend:
1960 	cpufreq_suspended = true;
1961 }
1962 
1963 /**
1964  * cpufreq_resume() - Resume CPUFreq governors.
1965  *
1966  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1967  * are suspended with cpufreq_suspend().
1968  */
cpufreq_resume(void)1969 void cpufreq_resume(void)
1970 {
1971 	struct cpufreq_policy *policy;
1972 	int ret;
1973 
1974 	if (!cpufreq_driver)
1975 		return;
1976 
1977 	if (unlikely(!cpufreq_suspended))
1978 		return;
1979 
1980 	cpufreq_suspended = false;
1981 
1982 	if (!has_target() && !cpufreq_driver->resume)
1983 		return;
1984 
1985 	pr_debug("%s: Resuming Governors\n", __func__);
1986 
1987 	for_each_active_policy(policy) {
1988 		if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1989 			pr_err("%s: Failed to resume driver: %s\n", __func__,
1990 				cpufreq_driver->name);
1991 		} else if (has_target()) {
1992 			down_write(&policy->rwsem);
1993 			ret = cpufreq_start_governor(policy);
1994 			up_write(&policy->rwsem);
1995 
1996 			if (ret)
1997 				pr_err("%s: Failed to start governor for CPU%u's policy\n",
1998 				       __func__, policy->cpu);
1999 		}
2000 	}
2001 }
2002 
2003 /**
2004  * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2005  * @flags: Flags to test against the current cpufreq driver's flags.
2006  *
2007  * Assumes that the driver is there, so callers must ensure that this is the
2008  * case.
2009  */
cpufreq_driver_test_flags(u16 flags)2010 bool cpufreq_driver_test_flags(u16 flags)
2011 {
2012 	return !!(cpufreq_driver->flags & flags);
2013 }
2014 
2015 /**
2016  * cpufreq_get_current_driver - Return the current driver's name.
2017  *
2018  * Return the name string of the currently registered cpufreq driver or NULL if
2019  * none.
2020  */
cpufreq_get_current_driver(void)2021 const char *cpufreq_get_current_driver(void)
2022 {
2023 	if (cpufreq_driver)
2024 		return cpufreq_driver->name;
2025 
2026 	return NULL;
2027 }
2028 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2029 
2030 /**
2031  * cpufreq_get_driver_data - Return current driver data.
2032  *
2033  * Return the private data of the currently registered cpufreq driver, or NULL
2034  * if no cpufreq driver has been registered.
2035  */
cpufreq_get_driver_data(void)2036 void *cpufreq_get_driver_data(void)
2037 {
2038 	if (cpufreq_driver)
2039 		return cpufreq_driver->driver_data;
2040 
2041 	return NULL;
2042 }
2043 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2044 
2045 /*********************************************************************
2046  *                     NOTIFIER LISTS INTERFACE                      *
2047  *********************************************************************/
2048 
2049 /**
2050  * cpufreq_register_notifier - Register a notifier with cpufreq.
2051  * @nb: notifier function to register.
2052  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2053  *
2054  * Add a notifier to one of two lists: either a list of notifiers that run on
2055  * clock rate changes (once before and once after every transition), or a list
2056  * of notifiers that ron on cpufreq policy changes.
2057  *
2058  * This function may sleep and it has the same return values as
2059  * blocking_notifier_chain_register().
2060  */
cpufreq_register_notifier(struct notifier_block * nb,unsigned int list)2061 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2062 {
2063 	int ret;
2064 
2065 	if (cpufreq_disabled())
2066 		return -EINVAL;
2067 
2068 	switch (list) {
2069 	case CPUFREQ_TRANSITION_NOTIFIER:
2070 		mutex_lock(&cpufreq_fast_switch_lock);
2071 
2072 		if (cpufreq_fast_switch_count > 0) {
2073 			mutex_unlock(&cpufreq_fast_switch_lock);
2074 			return -EBUSY;
2075 		}
2076 		ret = srcu_notifier_chain_register(
2077 				&cpufreq_transition_notifier_list, nb);
2078 		if (!ret)
2079 			cpufreq_fast_switch_count--;
2080 
2081 		mutex_unlock(&cpufreq_fast_switch_lock);
2082 		break;
2083 	case CPUFREQ_POLICY_NOTIFIER:
2084 		ret = blocking_notifier_chain_register(
2085 				&cpufreq_policy_notifier_list, nb);
2086 		break;
2087 	default:
2088 		ret = -EINVAL;
2089 	}
2090 
2091 	return ret;
2092 }
2093 EXPORT_SYMBOL(cpufreq_register_notifier);
2094 
2095 /**
2096  * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2097  * @nb: notifier block to be unregistered.
2098  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2099  *
2100  * Remove a notifier from one of the cpufreq notifier lists.
2101  *
2102  * This function may sleep and it has the same return values as
2103  * blocking_notifier_chain_unregister().
2104  */
cpufreq_unregister_notifier(struct notifier_block * nb,unsigned int list)2105 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2106 {
2107 	int ret;
2108 
2109 	if (cpufreq_disabled())
2110 		return -EINVAL;
2111 
2112 	switch (list) {
2113 	case CPUFREQ_TRANSITION_NOTIFIER:
2114 		mutex_lock(&cpufreq_fast_switch_lock);
2115 
2116 		ret = srcu_notifier_chain_unregister(
2117 				&cpufreq_transition_notifier_list, nb);
2118 		if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2119 			cpufreq_fast_switch_count++;
2120 
2121 		mutex_unlock(&cpufreq_fast_switch_lock);
2122 		break;
2123 	case CPUFREQ_POLICY_NOTIFIER:
2124 		ret = blocking_notifier_chain_unregister(
2125 				&cpufreq_policy_notifier_list, nb);
2126 		break;
2127 	default:
2128 		ret = -EINVAL;
2129 	}
2130 
2131 	return ret;
2132 }
2133 EXPORT_SYMBOL(cpufreq_unregister_notifier);
2134 
2135 
2136 /*********************************************************************
2137  *                              GOVERNORS                            *
2138  *********************************************************************/
2139 
2140 /**
2141  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2142  * @policy: cpufreq policy to switch the frequency for.
2143  * @target_freq: New frequency to set (may be approximate).
2144  *
2145  * Carry out a fast frequency switch without sleeping.
2146  *
2147  * The driver's ->fast_switch() callback invoked by this function must be
2148  * suitable for being called from within RCU-sched read-side critical sections
2149  * and it is expected to select the minimum available frequency greater than or
2150  * equal to @target_freq (CPUFREQ_RELATION_L).
2151  *
2152  * This function must not be called if policy->fast_switch_enabled is unset.
2153  *
2154  * Governors calling this function must guarantee that it will never be invoked
2155  * twice in parallel for the same policy and that it will never be called in
2156  * parallel with either ->target() or ->target_index() for the same policy.
2157  *
2158  * Returns the actual frequency set for the CPU.
2159  *
2160  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2161  * error condition, the hardware configuration must be preserved.
2162  */
cpufreq_driver_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)2163 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2164 					unsigned int target_freq)
2165 {
2166 	unsigned int freq;
2167 	int cpu;
2168 
2169 	target_freq = clamp_val(target_freq, policy->min, policy->max);
2170 	freq = cpufreq_driver->fast_switch(policy, target_freq);
2171 
2172 	if (!freq)
2173 		return 0;
2174 
2175 	policy->cur = freq;
2176 	arch_set_freq_scale(policy->related_cpus, freq,
2177 			    policy->cpuinfo.max_freq);
2178 	cpufreq_stats_record_transition(policy, freq);
2179 
2180 	if (trace_cpu_frequency_enabled()) {
2181 		for_each_cpu(cpu, policy->cpus)
2182 			trace_cpu_frequency(freq, cpu);
2183 	}
2184 
2185 	return freq;
2186 }
2187 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2188 
2189 /**
2190  * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2191  * @cpu: Target CPU.
2192  * @min_perf: Minimum (required) performance level (units of @capacity).
2193  * @target_perf: Target (desired) performance level (units of @capacity).
2194  * @capacity: Capacity of the target CPU.
2195  *
2196  * Carry out a fast performance level switch of @cpu without sleeping.
2197  *
2198  * The driver's ->adjust_perf() callback invoked by this function must be
2199  * suitable for being called from within RCU-sched read-side critical sections
2200  * and it is expected to select a suitable performance level equal to or above
2201  * @min_perf and preferably equal to or below @target_perf.
2202  *
2203  * This function must not be called if policy->fast_switch_enabled is unset.
2204  *
2205  * Governors calling this function must guarantee that it will never be invoked
2206  * twice in parallel for the same CPU and that it will never be called in
2207  * parallel with either ->target() or ->target_index() or ->fast_switch() for
2208  * the same CPU.
2209  */
cpufreq_driver_adjust_perf(unsigned int cpu,unsigned long min_perf,unsigned long target_perf,unsigned long capacity)2210 void cpufreq_driver_adjust_perf(unsigned int cpu,
2211 				 unsigned long min_perf,
2212 				 unsigned long target_perf,
2213 				 unsigned long capacity)
2214 {
2215 	cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
2216 }
2217 
2218 /**
2219  * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2220  *
2221  * Return 'true' if the ->adjust_perf callback is present for the
2222  * current driver or 'false' otherwise.
2223  */
cpufreq_driver_has_adjust_perf(void)2224 bool cpufreq_driver_has_adjust_perf(void)
2225 {
2226 	return !!cpufreq_driver->adjust_perf;
2227 }
2228 
2229 /* Must set freqs->new to intermediate frequency */
__target_intermediate(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int index)2230 static int __target_intermediate(struct cpufreq_policy *policy,
2231 				 struct cpufreq_freqs *freqs, int index)
2232 {
2233 	int ret;
2234 
2235 	freqs->new = cpufreq_driver->get_intermediate(policy, index);
2236 
2237 	/* We don't need to switch to intermediate freq */
2238 	if (!freqs->new)
2239 		return 0;
2240 
2241 	pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2242 		 __func__, policy->cpu, freqs->old, freqs->new);
2243 
2244 	cpufreq_freq_transition_begin(policy, freqs);
2245 	ret = cpufreq_driver->target_intermediate(policy, index);
2246 	cpufreq_freq_transition_end(policy, freqs, ret);
2247 
2248 	if (ret)
2249 		pr_err("%s: Failed to change to intermediate frequency: %d\n",
2250 		       __func__, ret);
2251 
2252 	return ret;
2253 }
2254 
__target_index(struct cpufreq_policy * policy,int index)2255 static int __target_index(struct cpufreq_policy *policy, int index)
2256 {
2257 	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2258 	unsigned int restore_freq, intermediate_freq = 0;
2259 	unsigned int newfreq = policy->freq_table[index].frequency;
2260 	int retval = -EINVAL;
2261 	bool notify;
2262 
2263 	if (newfreq == policy->cur)
2264 		return 0;
2265 
2266 	/* Save last value to restore later on errors */
2267 	restore_freq = policy->cur;
2268 
2269 	notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2270 	if (notify) {
2271 		/* Handle switching to intermediate frequency */
2272 		if (cpufreq_driver->get_intermediate) {
2273 			retval = __target_intermediate(policy, &freqs, index);
2274 			if (retval)
2275 				return retval;
2276 
2277 			intermediate_freq = freqs.new;
2278 			/* Set old freq to intermediate */
2279 			if (intermediate_freq)
2280 				freqs.old = freqs.new;
2281 		}
2282 
2283 		freqs.new = newfreq;
2284 		pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2285 			 __func__, policy->cpu, freqs.old, freqs.new);
2286 
2287 		cpufreq_freq_transition_begin(policy, &freqs);
2288 	}
2289 
2290 	retval = cpufreq_driver->target_index(policy, index);
2291 	if (retval)
2292 		pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2293 		       retval);
2294 
2295 	if (notify) {
2296 		cpufreq_freq_transition_end(policy, &freqs, retval);
2297 
2298 		/*
2299 		 * Failed after setting to intermediate freq? Driver should have
2300 		 * reverted back to initial frequency and so should we. Check
2301 		 * here for intermediate_freq instead of get_intermediate, in
2302 		 * case we haven't switched to intermediate freq at all.
2303 		 */
2304 		if (unlikely(retval && intermediate_freq)) {
2305 			freqs.old = intermediate_freq;
2306 			freqs.new = restore_freq;
2307 			cpufreq_freq_transition_begin(policy, &freqs);
2308 			cpufreq_freq_transition_end(policy, &freqs, 0);
2309 		}
2310 	}
2311 
2312 	return retval;
2313 }
2314 
__cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2315 int __cpufreq_driver_target(struct cpufreq_policy *policy,
2316 			    unsigned int target_freq,
2317 			    unsigned int relation)
2318 {
2319 	unsigned int old_target_freq = target_freq;
2320 
2321 	if (cpufreq_disabled())
2322 		return -ENODEV;
2323 
2324 	target_freq = __resolve_freq(policy, target_freq, relation);
2325 
2326 	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
2327 		 policy->cpu, target_freq, relation, old_target_freq);
2328 
2329 	/*
2330 	 * This might look like a redundant call as we are checking it again
2331 	 * after finding index. But it is left intentionally for cases where
2332 	 * exactly same freq is called again and so we can save on few function
2333 	 * calls.
2334 	 */
2335 	if (target_freq == policy->cur &&
2336 	    !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
2337 		return 0;
2338 
2339 	if (cpufreq_driver->target) {
2340 		/*
2341 		 * If the driver hasn't setup a single inefficient frequency,
2342 		 * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2343 		 */
2344 		if (!policy->efficiencies_available)
2345 			relation &= ~CPUFREQ_RELATION_E;
2346 
2347 		return cpufreq_driver->target(policy, target_freq, relation);
2348 	}
2349 
2350 	if (!cpufreq_driver->target_index)
2351 		return -EINVAL;
2352 
2353 	return __target_index(policy, policy->cached_resolved_idx);
2354 }
2355 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2356 
cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2357 int cpufreq_driver_target(struct cpufreq_policy *policy,
2358 			  unsigned int target_freq,
2359 			  unsigned int relation)
2360 {
2361 	int ret;
2362 
2363 	down_write(&policy->rwsem);
2364 
2365 	ret = __cpufreq_driver_target(policy, target_freq, relation);
2366 
2367 	up_write(&policy->rwsem);
2368 
2369 	return ret;
2370 }
2371 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2372 
cpufreq_fallback_governor(void)2373 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2374 {
2375 	return NULL;
2376 }
2377 
cpufreq_init_governor(struct cpufreq_policy * policy)2378 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2379 {
2380 	int ret;
2381 
2382 	/* Don't start any governor operations if we are entering suspend */
2383 	if (cpufreq_suspended)
2384 		return 0;
2385 	/*
2386 	 * Governor might not be initiated here if ACPI _PPC changed
2387 	 * notification happened, so check it.
2388 	 */
2389 	if (!policy->governor)
2390 		return -EINVAL;
2391 
2392 	/* Platform doesn't want dynamic frequency switching ? */
2393 	if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2394 	    cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2395 		struct cpufreq_governor *gov = cpufreq_fallback_governor();
2396 
2397 		if (gov) {
2398 			pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2399 				policy->governor->name, gov->name);
2400 			policy->governor = gov;
2401 		} else {
2402 			return -EINVAL;
2403 		}
2404 	}
2405 
2406 	if (!try_module_get(policy->governor->owner))
2407 		return -EINVAL;
2408 
2409 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2410 
2411 	if (policy->governor->init) {
2412 		ret = policy->governor->init(policy);
2413 		if (ret) {
2414 			module_put(policy->governor->owner);
2415 			return ret;
2416 		}
2417 	}
2418 
2419 	policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2420 
2421 	return 0;
2422 }
2423 
cpufreq_exit_governor(struct cpufreq_policy * policy)2424 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2425 {
2426 	if (cpufreq_suspended || !policy->governor)
2427 		return;
2428 
2429 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2430 
2431 	if (policy->governor->exit)
2432 		policy->governor->exit(policy);
2433 
2434 	module_put(policy->governor->owner);
2435 }
2436 
cpufreq_start_governor(struct cpufreq_policy * policy)2437 int cpufreq_start_governor(struct cpufreq_policy *policy)
2438 {
2439 	int ret;
2440 
2441 	if (cpufreq_suspended)
2442 		return 0;
2443 
2444 	if (!policy->governor)
2445 		return -EINVAL;
2446 
2447 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2448 
2449 	if (cpufreq_driver->get)
2450 		cpufreq_verify_current_freq(policy, false);
2451 
2452 	if (policy->governor->start) {
2453 		ret = policy->governor->start(policy);
2454 		if (ret)
2455 			return ret;
2456 	}
2457 
2458 	if (policy->governor->limits)
2459 		policy->governor->limits(policy);
2460 
2461 	return 0;
2462 }
2463 
cpufreq_stop_governor(struct cpufreq_policy * policy)2464 void cpufreq_stop_governor(struct cpufreq_policy *policy)
2465 {
2466 	if (cpufreq_suspended || !policy->governor)
2467 		return;
2468 
2469 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2470 
2471 	if (policy->governor->stop)
2472 		policy->governor->stop(policy);
2473 }
2474 
cpufreq_governor_limits(struct cpufreq_policy * policy)2475 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2476 {
2477 	if (cpufreq_suspended || !policy->governor)
2478 		return;
2479 
2480 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2481 
2482 	if (policy->governor->limits)
2483 		policy->governor->limits(policy);
2484 }
2485 
cpufreq_register_governor(struct cpufreq_governor * governor)2486 int cpufreq_register_governor(struct cpufreq_governor *governor)
2487 {
2488 	int err;
2489 
2490 	if (!governor)
2491 		return -EINVAL;
2492 
2493 	if (cpufreq_disabled())
2494 		return -ENODEV;
2495 
2496 	mutex_lock(&cpufreq_governor_mutex);
2497 
2498 	err = -EBUSY;
2499 	if (!find_governor(governor->name)) {
2500 		err = 0;
2501 		list_add(&governor->governor_list, &cpufreq_governor_list);
2502 	}
2503 
2504 	mutex_unlock(&cpufreq_governor_mutex);
2505 	return err;
2506 }
2507 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2508 
cpufreq_unregister_governor(struct cpufreq_governor * governor)2509 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2510 {
2511 	struct cpufreq_policy *policy;
2512 	unsigned long flags;
2513 
2514 	if (!governor)
2515 		return;
2516 
2517 	if (cpufreq_disabled())
2518 		return;
2519 
2520 	/* clear last_governor for all inactive policies */
2521 	read_lock_irqsave(&cpufreq_driver_lock, flags);
2522 	for_each_inactive_policy(policy) {
2523 		if (!strcmp(policy->last_governor, governor->name)) {
2524 			policy->governor = NULL;
2525 			strcpy(policy->last_governor, "\0");
2526 		}
2527 	}
2528 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2529 
2530 	mutex_lock(&cpufreq_governor_mutex);
2531 	list_del(&governor->governor_list);
2532 	mutex_unlock(&cpufreq_governor_mutex);
2533 }
2534 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2535 
2536 
2537 /*********************************************************************
2538  *                          POLICY INTERFACE                         *
2539  *********************************************************************/
2540 
2541 /**
2542  * cpufreq_get_policy - get the current cpufreq_policy
2543  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2544  *	is written
2545  * @cpu: CPU to find the policy for
2546  *
2547  * Reads the current cpufreq policy.
2548  */
cpufreq_get_policy(struct cpufreq_policy * policy,unsigned int cpu)2549 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2550 {
2551 	struct cpufreq_policy *cpu_policy;
2552 	if (!policy)
2553 		return -EINVAL;
2554 
2555 	cpu_policy = cpufreq_cpu_get(cpu);
2556 	if (!cpu_policy)
2557 		return -EINVAL;
2558 
2559 	memcpy(policy, cpu_policy, sizeof(*policy));
2560 
2561 	cpufreq_cpu_put(cpu_policy);
2562 	return 0;
2563 }
2564 EXPORT_SYMBOL(cpufreq_get_policy);
2565 
2566 /**
2567  * cpufreq_set_policy - Modify cpufreq policy parameters.
2568  * @policy: Policy object to modify.
2569  * @new_gov: Policy governor pointer.
2570  * @new_pol: Policy value (for drivers with built-in governors).
2571  *
2572  * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2573  * limits to be set for the policy, update @policy with the verified limits
2574  * values and either invoke the driver's ->setpolicy() callback (if present) or
2575  * carry out a governor update for @policy.  That is, run the current governor's
2576  * ->limits() callback (if @new_gov points to the same object as the one in
2577  * @policy) or replace the governor for @policy with @new_gov.
2578  *
2579  * The cpuinfo part of @policy is not updated by this function.
2580  */
cpufreq_set_policy(struct cpufreq_policy * policy,struct cpufreq_governor * new_gov,unsigned int new_pol)2581 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2582 			      struct cpufreq_governor *new_gov,
2583 			      unsigned int new_pol)
2584 {
2585 	struct cpufreq_policy_data new_data;
2586 	struct cpufreq_governor *old_gov;
2587 	int ret;
2588 
2589 	memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2590 	new_data.freq_table = policy->freq_table;
2591 	new_data.cpu = policy->cpu;
2592 	/*
2593 	 * PM QoS framework collects all the requests from users and provide us
2594 	 * the final aggregated value here.
2595 	 */
2596 	new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2597 	new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2598 
2599 	pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2600 		 new_data.cpu, new_data.min, new_data.max);
2601 
2602 	/*
2603 	 * Verify that the CPU speed can be set within these limits and make sure
2604 	 * that min <= max.
2605 	 */
2606 	ret = cpufreq_driver->verify(&new_data);
2607 	if (ret)
2608 		return ret;
2609 
2610 	/*
2611 	 * Resolve policy min/max to available frequencies. It ensures
2612 	 * no frequency resolution will neither overshoot the requested maximum
2613 	 * nor undershoot the requested minimum.
2614 	 */
2615 	policy->min = new_data.min;
2616 	policy->max = new_data.max;
2617 	policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
2618 	policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
2619 	trace_cpu_frequency_limits(policy);
2620 
2621 	policy->cached_target_freq = UINT_MAX;
2622 
2623 	pr_debug("new min and max freqs are %u - %u kHz\n",
2624 		 policy->min, policy->max);
2625 
2626 	if (cpufreq_driver->setpolicy) {
2627 		policy->policy = new_pol;
2628 		pr_debug("setting range\n");
2629 		return cpufreq_driver->setpolicy(policy);
2630 	}
2631 
2632 	if (new_gov == policy->governor) {
2633 		pr_debug("governor limits update\n");
2634 		cpufreq_governor_limits(policy);
2635 		return 0;
2636 	}
2637 
2638 	pr_debug("governor switch\n");
2639 
2640 	/* save old, working values */
2641 	old_gov = policy->governor;
2642 	/* end old governor */
2643 	if (old_gov) {
2644 		cpufreq_stop_governor(policy);
2645 		cpufreq_exit_governor(policy);
2646 	}
2647 
2648 	/* start new governor */
2649 	policy->governor = new_gov;
2650 	ret = cpufreq_init_governor(policy);
2651 	if (!ret) {
2652 		ret = cpufreq_start_governor(policy);
2653 		if (!ret) {
2654 			pr_debug("governor change\n");
2655 			sched_cpufreq_governor_change(policy, old_gov);
2656 			return 0;
2657 		}
2658 		cpufreq_exit_governor(policy);
2659 	}
2660 
2661 	/* new governor failed, so re-start old one */
2662 	pr_debug("starting governor %s failed\n", policy->governor->name);
2663 	if (old_gov) {
2664 		policy->governor = old_gov;
2665 		if (cpufreq_init_governor(policy))
2666 			policy->governor = NULL;
2667 		else
2668 			cpufreq_start_governor(policy);
2669 	}
2670 
2671 	return ret;
2672 }
2673 
2674 /**
2675  * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2676  * @cpu: CPU to re-evaluate the policy for.
2677  *
2678  * Update the current frequency for the cpufreq policy of @cpu and use
2679  * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2680  * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2681  * for the policy in question, among other things.
2682  */
cpufreq_update_policy(unsigned int cpu)2683 void cpufreq_update_policy(unsigned int cpu)
2684 {
2685 	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
2686 
2687 	if (!policy)
2688 		return;
2689 
2690 	/*
2691 	 * BIOS might change freq behind our back
2692 	 * -> ask driver for current freq and notify governors about a change
2693 	 */
2694 	if (cpufreq_driver->get && has_target() &&
2695 	    (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2696 		goto unlock;
2697 
2698 	refresh_frequency_limits(policy);
2699 
2700 unlock:
2701 	cpufreq_cpu_release(policy);
2702 }
2703 EXPORT_SYMBOL(cpufreq_update_policy);
2704 
2705 /**
2706  * cpufreq_update_limits - Update policy limits for a given CPU.
2707  * @cpu: CPU to update the policy limits for.
2708  *
2709  * Invoke the driver's ->update_limits callback if present or call
2710  * cpufreq_update_policy() for @cpu.
2711  */
cpufreq_update_limits(unsigned int cpu)2712 void cpufreq_update_limits(unsigned int cpu)
2713 {
2714 	if (cpufreq_driver->update_limits)
2715 		cpufreq_driver->update_limits(cpu);
2716 	else
2717 		cpufreq_update_policy(cpu);
2718 }
2719 EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2720 
2721 /*********************************************************************
2722  *               BOOST						     *
2723  *********************************************************************/
cpufreq_boost_set_sw(struct cpufreq_policy * policy,int state)2724 static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2725 {
2726 	int ret;
2727 
2728 	if (!policy->freq_table)
2729 		return -ENXIO;
2730 
2731 	ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
2732 	if (ret) {
2733 		pr_err("%s: Policy frequency update failed\n", __func__);
2734 		return ret;
2735 	}
2736 
2737 	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
2738 	if (ret < 0)
2739 		return ret;
2740 
2741 	return 0;
2742 }
2743 
cpufreq_boost_trigger_state(int state)2744 int cpufreq_boost_trigger_state(int state)
2745 {
2746 	struct cpufreq_policy *policy;
2747 	unsigned long flags;
2748 	int ret = 0;
2749 
2750 	if (cpufreq_driver->boost_enabled == state)
2751 		return 0;
2752 
2753 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2754 	cpufreq_driver->boost_enabled = state;
2755 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2756 
2757 	cpus_read_lock();
2758 	for_each_active_policy(policy) {
2759 		ret = cpufreq_driver->set_boost(policy, state);
2760 		if (ret)
2761 			goto err_reset_state;
2762 
2763 		policy->boost_enabled = state;
2764 	}
2765 	cpus_read_unlock();
2766 
2767 	return 0;
2768 
2769 err_reset_state:
2770 	cpus_read_unlock();
2771 
2772 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2773 	cpufreq_driver->boost_enabled = !state;
2774 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2775 
2776 	pr_err("%s: Cannot %s BOOST\n",
2777 	       __func__, state ? "enable" : "disable");
2778 
2779 	return ret;
2780 }
2781 
cpufreq_boost_supported(void)2782 static bool cpufreq_boost_supported(void)
2783 {
2784 	return cpufreq_driver->set_boost;
2785 }
2786 
create_boost_sysfs_file(void)2787 static int create_boost_sysfs_file(void)
2788 {
2789 	int ret;
2790 
2791 	ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2792 	if (ret)
2793 		pr_err("%s: cannot register global BOOST sysfs file\n",
2794 		       __func__);
2795 
2796 	return ret;
2797 }
2798 
remove_boost_sysfs_file(void)2799 static void remove_boost_sysfs_file(void)
2800 {
2801 	if (cpufreq_boost_supported())
2802 		sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2803 }
2804 
cpufreq_enable_boost_support(void)2805 int cpufreq_enable_boost_support(void)
2806 {
2807 	if (!cpufreq_driver)
2808 		return -EINVAL;
2809 
2810 	if (cpufreq_boost_supported())
2811 		return 0;
2812 
2813 	cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2814 
2815 	/* This will get removed on driver unregister */
2816 	return create_boost_sysfs_file();
2817 }
2818 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2819 
cpufreq_boost_enabled(void)2820 int cpufreq_boost_enabled(void)
2821 {
2822 	return cpufreq_driver->boost_enabled;
2823 }
2824 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2825 
2826 /*********************************************************************
2827  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2828  *********************************************************************/
2829 static enum cpuhp_state hp_online;
2830 
cpuhp_cpufreq_online(unsigned int cpu)2831 static int cpuhp_cpufreq_online(unsigned int cpu)
2832 {
2833 	cpufreq_online(cpu);
2834 
2835 	return 0;
2836 }
2837 
cpuhp_cpufreq_offline(unsigned int cpu)2838 static int cpuhp_cpufreq_offline(unsigned int cpu)
2839 {
2840 	cpufreq_offline(cpu);
2841 
2842 	return 0;
2843 }
2844 
2845 /**
2846  * cpufreq_register_driver - register a CPU Frequency driver
2847  * @driver_data: A struct cpufreq_driver containing the values#
2848  * submitted by the CPU Frequency driver.
2849  *
2850  * Registers a CPU Frequency driver to this core code. This code
2851  * returns zero on success, -EEXIST when another driver got here first
2852  * (and isn't unregistered in the meantime).
2853  *
2854  */
cpufreq_register_driver(struct cpufreq_driver * driver_data)2855 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2856 {
2857 	unsigned long flags;
2858 	int ret;
2859 
2860 	if (cpufreq_disabled())
2861 		return -ENODEV;
2862 
2863 	/*
2864 	 * The cpufreq core depends heavily on the availability of device
2865 	 * structure, make sure they are available before proceeding further.
2866 	 */
2867 	if (!get_cpu_device(0))
2868 		return -EPROBE_DEFER;
2869 
2870 	if (!driver_data || !driver_data->verify || !driver_data->init ||
2871 	    !(driver_data->setpolicy || driver_data->target_index ||
2872 		    driver_data->target) ||
2873 	     (driver_data->setpolicy && (driver_data->target_index ||
2874 		    driver_data->target)) ||
2875 	     (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2876 	     (!driver_data->online != !driver_data->offline) ||
2877 		 (driver_data->adjust_perf && !driver_data->fast_switch))
2878 		return -EINVAL;
2879 
2880 	pr_debug("trying to register driver %s\n", driver_data->name);
2881 
2882 	/* Protect against concurrent CPU online/offline. */
2883 	cpus_read_lock();
2884 
2885 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2886 	if (cpufreq_driver) {
2887 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2888 		ret = -EEXIST;
2889 		goto out;
2890 	}
2891 	cpufreq_driver = driver_data;
2892 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2893 
2894 	/*
2895 	 * Mark support for the scheduler's frequency invariance engine for
2896 	 * drivers that implement target(), target_index() or fast_switch().
2897 	 */
2898 	if (!cpufreq_driver->setpolicy) {
2899 		static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2900 		pr_debug("supports frequency invariance");
2901 	}
2902 
2903 	if (driver_data->setpolicy)
2904 		driver_data->flags |= CPUFREQ_CONST_LOOPS;
2905 
2906 	if (cpufreq_boost_supported()) {
2907 		ret = create_boost_sysfs_file();
2908 		if (ret)
2909 			goto err_null_driver;
2910 	}
2911 
2912 	ret = subsys_interface_register(&cpufreq_interface);
2913 	if (ret)
2914 		goto err_boost_unreg;
2915 
2916 	if (unlikely(list_empty(&cpufreq_policy_list))) {
2917 		/* if all ->init() calls failed, unregister */
2918 		ret = -ENODEV;
2919 		pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2920 			 driver_data->name);
2921 		goto err_if_unreg;
2922 	}
2923 
2924 	ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2925 						   "cpufreq:online",
2926 						   cpuhp_cpufreq_online,
2927 						   cpuhp_cpufreq_offline);
2928 	if (ret < 0)
2929 		goto err_if_unreg;
2930 	hp_online = ret;
2931 	ret = 0;
2932 
2933 	pr_debug("driver %s up and running\n", driver_data->name);
2934 	goto out;
2935 
2936 err_if_unreg:
2937 	subsys_interface_unregister(&cpufreq_interface);
2938 err_boost_unreg:
2939 	remove_boost_sysfs_file();
2940 err_null_driver:
2941 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2942 	cpufreq_driver = NULL;
2943 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2944 out:
2945 	cpus_read_unlock();
2946 	return ret;
2947 }
2948 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2949 
2950 /*
2951  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2952  *
2953  * Unregister the current CPUFreq driver. Only call this if you have
2954  * the right to do so, i.e. if you have succeeded in initialising before!
2955  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2956  * currently not initialised.
2957  */
cpufreq_unregister_driver(struct cpufreq_driver * driver)2958 void cpufreq_unregister_driver(struct cpufreq_driver *driver)
2959 {
2960 	unsigned long flags;
2961 
2962 	if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
2963 		return;
2964 
2965 	pr_debug("unregistering driver %s\n", driver->name);
2966 
2967 	/* Protect against concurrent cpu hotplug */
2968 	cpus_read_lock();
2969 	subsys_interface_unregister(&cpufreq_interface);
2970 	remove_boost_sysfs_file();
2971 	static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
2972 	cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2973 
2974 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2975 
2976 	cpufreq_driver = NULL;
2977 
2978 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2979 	cpus_read_unlock();
2980 }
2981 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2982 
cpufreq_core_init(void)2983 static int __init cpufreq_core_init(void)
2984 {
2985 	struct cpufreq_governor *gov = cpufreq_default_governor();
2986 	struct device *dev_root;
2987 
2988 	if (cpufreq_disabled())
2989 		return -ENODEV;
2990 
2991 	dev_root = bus_get_dev_root(&cpu_subsys);
2992 	if (dev_root) {
2993 		cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
2994 		put_device(dev_root);
2995 	}
2996 	BUG_ON(!cpufreq_global_kobject);
2997 
2998 	if (!strlen(default_governor))
2999 		strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
3000 
3001 	return 0;
3002 }
3003 module_param(off, int, 0444);
3004 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3005 core_initcall(cpufreq_core_init);
3006