drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  42 {
  43         return active == !policy_is_inactive(policy);
  44 }
  45
  46 /* Finds Next Acive/Inactive policy */
  47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  48                                           bool active)
  49 {
  50         do {
  51                 policy = list_next_entry(policy, policy_list);
  52
  53                 /* No more policies in the list */
  54                 if (&policy->policy_list == &cpufreq_policy_list)
  55                         return NULL;
  56         } while (!suitable_policy(policy, active));
  57
  58         return policy;
  59 }
  60
  61 static struct cpufreq_policy *first_policy(bool active)
  62 {
  63         struct cpufreq_policy *policy;
  64
  65         /* No policies in the list */
  66         if (list_empty(&cpufreq_policy_list))
  67                 return NULL;
  68
  69         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  70                                   policy_list);
  71
  72         if (!suitable_policy(policy, active))
  73                 policy = next_policy(policy, active);
  74
  75         return policy;
  76 }
  77
  78 /* Macros to iterate over CPU policies */
  79 #define for_each_suitable_policy(__policy, __active)    \
  80         for (__policy = first_policy(__active);         \
  81              __policy;                                  \
  82              __policy = next_policy(__policy, __active))
  83
  84 #define for_each_active_policy(__policy)                \
  85         for_each_suitable_policy(__policy, true)
  86 #define for_each_inactive_policy(__policy)              \
  87         for_each_suitable_policy(__policy, false)
  88
  89 #define for_each_policy(__policy)                       \
  90         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  91
  92 /* Iterate over governors */
  93 static LIST_HEAD(cpufreq_governor_list);
  94 #define for_each_governor(__governor)                           \
  95         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  96
  97 /**
  98  * The "cpufreq driver" - the arch- or hardware-dependent low
  99  * level driver of CPUFreq support, and its spinlock. This lock
 100  * also protects the cpufreq_cpu_data array.
 101  */
 102 static struct cpufreq_driver *cpufreq_driver;
 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 104 static DEFINE_RWLOCK(cpufreq_driver_lock);
 105 DEFINE_MUTEX(cpufreq_governor_lock);
 106
 107 /* Flag to suspend/resume CPUFreq governors */
 108 static bool cpufreq_suspended;
 109
 110 static inline bool has_target(void)
 111 {
 112         return cpufreq_driver->target_index || cpufreq_driver->target;
 113 }
 114
 115 /* internal prototypes */
 116 static int __cpufreq_governor(struct cpufreq_policy *policy,
 117                 unsigned int event);
 118 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 119 static void handle_update(struct work_struct *work);
 120
 121 /**
 122  * Two notifier lists: the "policy" list is involved in the
 123  * validation process for a new CPU frequency policy; the
 124  * "transition" list for kernel code that needs to handle
 125  * changes to devices when the CPU clock speed changes.
 126  * The mutex locks both lists.
 127  */
 128 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 129 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 130
 131 static bool init_cpufreq_transition_notifier_list_called;
 132 static int __init init_cpufreq_transition_notifier_list(void)
 133 {
 134         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 135         init_cpufreq_transition_notifier_list_called = true;
 136         return 0;
 137 }
 138 pure_initcall(init_cpufreq_transition_notifier_list);
 139
 140 static int off __read_mostly;
 141 static int cpufreq_disabled(void)
 142 {
 143         return off;
 144 }
 145 void disable_cpufreq(void)
 146 {
 147         off = 1;
 148 }
 149 static DEFINE_MUTEX(cpufreq_governor_mutex);
 150
 151 bool have_governor_per_policy(void)
 152 {
 153         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 154 }
 155 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 156
 157 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 158 {
 159         if (have_governor_per_policy())
 160                 return &policy->kobj;
 161         else
 162                 return cpufreq_global_kobject;
 163 }
 164 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 165
 166 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
 167 {
 168         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 169
 170         return policy && !policy_is_inactive(policy) ?
 171                 policy->freq_table : NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
 174
 175 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 176 {
 177         u64 idle_time;
 178         u64 cur_wall_time;
 179         u64 busy_time;
 180
 181         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 182
 183         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 184         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 185         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 186         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 187         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 188         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 189
 190         idle_time = cur_wall_time - busy_time;
 191         if (wall)
 192                 *wall = cputime_to_usecs(cur_wall_time);
 193
 194         return cputime_to_usecs(idle_time);
 195 }
 196
 197 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 198 {
 199         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 200
 201         if (idle_time == -1ULL)
 202                 return get_cpu_idle_time_jiffy(cpu, wall);
 203         else if (!io_busy)
 204                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 205
 206         return idle_time;
 207 }
 208 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 209
 210 /*
 211  * This is a generic cpufreq init() routine which can be used by cpufreq
 212  * drivers of SMP systems. It will do following:
 213  * - validate & show freq table passed
 214  * - set policies transition latency
 215  * - policy->cpus with all possible CPUs
 216  */
 217 int cpufreq_generic_init(struct cpufreq_policy *policy,
 218                 struct cpufreq_frequency_table *table,
 219                 unsigned int transition_latency)
 220 {
 221         int ret;
 222
 223         ret = cpufreq_table_validate_and_show(policy, table);
 224         if (ret) {
 225                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 226                 return ret;
 227         }
 228
 229         policy->cpuinfo.transition_latency = transition_latency;
 230
 231         /*
 232          * The driver only supports the SMP configuration where all processors
 233          * share the clock and voltage and clock.
 234          */
 235         cpumask_setall(policy->cpus);
 236
 237         return 0;
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 240
 241 /* Only for cpufreq core internal use */
 242 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 243 {
 244         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 245
 246         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 247 }
 248
 249 unsigned int cpufreq_generic_get(unsigned int cpu)
 250 {
 251         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 252
 253         if (!policy || IS_ERR(policy->clk)) {
 254                 pr_err("%s: No %s associated to cpu: %d\n",
 255                        __func__, policy ? "clk" : "policy", cpu);
 256                 return 0;
 257         }
 258
 259         return clk_get_rate(policy->clk) / 1000;
 260 }
 261 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 262
 263 /**
 264  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 265  *
 266  * @cpu: cpu to find policy for.
 267  *
 268  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 269  * It also increments the kobject reference count to mark it busy and so would
 270  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 271  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 272  * freed as that depends on the kobj count.
 273  *
 274  * Return: A valid policy on success, otherwise NULL on failure.
 275  */
 276 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 277 {
 278         struct cpufreq_policy *policy = NULL;
 279         unsigned long flags;
 280
 281         if (WARN_ON(cpu >= nr_cpu_ids))
 282                 return NULL;
 283
 284         /* get the cpufreq driver */
 285         read_lock_irqsave(&cpufreq_driver_lock, flags);
 286
 287         if (cpufreq_driver) {
 288                 /* get the CPU */
 289                 policy = cpufreq_cpu_get_raw(cpu);
 290                 if (policy)
 291                         kobject_get(&policy->kobj);
 292         }
 293
 294         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 295
 296         return policy;
 297 }
 298 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 299
 300 /**
 301  * cpufreq_cpu_put: Decrements the usage count of a policy
 302  *
 303  * @policy: policy earlier returned by cpufreq_cpu_get().
 304  *
 305  * This decrements the kobject reference count incremented earlier by calling
 306  * cpufreq_cpu_get().
 307  */
 308 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 309 {
 310         kobject_put(&policy->kobj);
 311 }
 312 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 313
 314 /*********************************************************************
 315  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 316  *********************************************************************/
 317
 318 /**
 319  * adjust_jiffies - adjust the system "loops_per_jiffy"
 320  *
 321  * This function alters the system "loops_per_jiffy" for the clock
 322  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 323  * systems as each CPU might be scaled differently. So, use the arch
 324  * per-CPU loops_per_jiffy value wherever possible.
 325  */
 326 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 327 {
 328 #ifndef CONFIG_SMP
 329         static unsigned long l_p_j_ref;
 330         static unsigned int l_p_j_ref_freq;
 331
 332         if (ci->flags & CPUFREQ_CONST_LOOPS)
 333                 return;
 334
 335         if (!l_p_j_ref_freq) {
 336                 l_p_j_ref = loops_per_jiffy;
 337                 l_p_j_ref_freq = ci->old;
 338                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 339                          l_p_j_ref, l_p_j_ref_freq);
 340         }
 341         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 342                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 343                                                                 ci->new);
 344                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 345                          loops_per_jiffy, ci->new);
 346         }
 347 #endif
 348 }
 349
 350 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 351                 struct cpufreq_freqs *freqs, unsigned int state)
 352 {
 353         BUG_ON(irqs_disabled());
 354
 355         if (cpufreq_disabled())
 356                 return;
 357
 358         freqs->flags = cpufreq_driver->flags;
 359         pr_debug("notification %u of frequency transition to %u kHz\n",
 360                  state, freqs->new);
 361
 362         switch (state) {
 363
 364         case CPUFREQ_PRECHANGE:
 365                 /* detect if the driver reported a value as "old frequency"
 366                  * which is not equal to what the cpufreq core thinks is
 367                  * "old frequency".
 368                  */
 369                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 370                         if ((policy) && (policy->cpu == freqs->cpu) &&
 371                             (policy->cur) && (policy->cur != freqs->old)) {
 372                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 373                                          freqs->old, policy->cur);
 374                                 freqs->old = policy->cur;
 375                         }
 376                 }
 377                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 378                                 CPUFREQ_PRECHANGE, freqs);
 379                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 380                 break;
 381
 382         case CPUFREQ_POSTCHANGE:
 383                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 384                 pr_debug("FREQ: %lu - CPU: %lu\n",
 385                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 386                 trace_cpu_frequency(freqs->new, freqs->cpu);
 387                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 388                                 CPUFREQ_POSTCHANGE, freqs);
 389                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 390                         policy->cur = freqs->new;
 391                 break;
 392         }
 393 }
 394
 395 /**
 396  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 397  * on frequency transition.
 398  *
 399  * This function calls the transition notifiers and the "adjust_jiffies"
 400  * function. It is called twice on all CPU frequency changes that have
 401  * external effects.
 402  */
 403 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, unsigned int state)
 405 {
 406         for_each_cpu(freqs->cpu, policy->cpus)
 407                 __cpufreq_notify_transition(policy, freqs, state);
 408 }
 409
 410 /* Do post notifications when there are chances that transition has failed */
 411 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 412                 struct cpufreq_freqs *freqs, int transition_failed)
 413 {
 414         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 415         if (!transition_failed)
 416                 return;
 417
 418         swap(freqs->old, freqs->new);
 419         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 420         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 421 }
 422
 423 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 424                 struct cpufreq_freqs *freqs)
 425 {
 426
 427         /*
 428          * Catch double invocations of _begin() which lead to self-deadlock.
 429          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 430          * doesn't invoke _begin() on their behalf, and hence the chances of
 431          * double invocations are very low. Moreover, there are scenarios
 432          * where these checks can emit false-positive warnings in these
 433          * drivers; so we avoid that by skipping them altogether.
 434          */
 435         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 436                                 && current == policy->transition_task);
 437
 438 wait:
 439         wait_event(policy->transition_wait, !policy->transition_ongoing);
 440
 441         spin_lock(&policy->transition_lock);
 442
 443         if (unlikely(policy->transition_ongoing)) {
 444                 spin_unlock(&policy->transition_lock);
 445                 goto wait;
 446         }
 447
 448         policy->transition_ongoing = true;
 449         policy->transition_task = current;
 450
 451         spin_unlock(&policy->transition_lock);
 452
 453         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 454 }
 455 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 456
 457 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 458                 struct cpufreq_freqs *freqs, int transition_failed)
 459 {
 460         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 461                 return;
 462
 463         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 464
 465         policy->transition_ongoing = false;
 466         policy->transition_task = NULL;
 467
 468         wake_up(&policy->transition_wait);
 469 }
 470 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 471
 472
 473 /*********************************************************************
 474  *                          SYSFS INTERFACE                          *
 475  *********************************************************************/
 476 static ssize_t show_boost(struct kobject *kobj,
 477                                  struct attribute *attr, char *buf)
 478 {
 479         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 480 }
 481
 482 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 483                                   const char *buf, size_t count)
 484 {
 485         int ret, enable;
 486
 487         ret = sscanf(buf, "%d", &enable);
 488         if (ret != 1 || enable < 0 || enable > 1)
 489                 return -EINVAL;
 490
 491         if (cpufreq_boost_trigger_state(enable)) {
 492                 pr_err("%s: Cannot %s BOOST!\n",
 493                        __func__, enable ? "enable" : "disable");
 494                 return -EINVAL;
 495         }
 496
 497         pr_debug("%s: cpufreq BOOST %s\n",
 498                  __func__, enable ? "enabled" : "disabled");
 499
 500         return count;
 501 }
 502 define_one_global_rw(boost);
 503
 504 static struct cpufreq_governor *find_governor(const char *str_governor)
 505 {
 506         struct cpufreq_governor *t;
 507
 508         for_each_governor(t)
 509                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 510                         return t;
 511
 512         return NULL;
 513 }
 514
 515 /**
 516  * cpufreq_parse_governor - parse a governor string
 517  */
 518 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 519                                 struct cpufreq_governor **governor)
 520 {
 521         int err = -EINVAL;
 522
 523         if (!cpufreq_driver)
 524                 goto out;
 525
 526         if (cpufreq_driver->setpolicy) {
 527                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 528                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 529                         err = 0;
 530                 } else if (!strncasecmp(str_governor, "powersave",
 531                                                 CPUFREQ_NAME_LEN)) {
 532                         *policy = CPUFREQ_POLICY_POWERSAVE;
 533                         err = 0;
 534                 }
 535         } else {
 536                 struct cpufreq_governor *t;
 537
 538                 mutex_lock(&cpufreq_governor_mutex);
 539
 540                 t = find_governor(str_governor);
 541
 542                 if (t == NULL) {
 543                         int ret;
 544
 545                         mutex_unlock(&cpufreq_governor_mutex);
 546                         ret = request_module("cpufreq_%s", str_governor);
 547                         mutex_lock(&cpufreq_governor_mutex);
 548
 549                         if (ret == 0)
 550                                 t = find_governor(str_governor);
 551                 }
 552
 553                 if (t != NULL) {
 554                         *governor = t;
 555                         err = 0;
 556                 }
 557
 558                 mutex_unlock(&cpufreq_governor_mutex);
 559         }
 560 out:
 561         return err;
 562 }
 563
 564 /**
 565  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 566  * print out cpufreq information
 567  *
 568  * Write out information from cpufreq_driver->policy[cpu]; object must be
 569  * "unsigned int".
 570  */
 571
 572 #define show_one(file_name, object)                     \
 573 static ssize_t show_##file_name                         \
 574 (struct cpufreq_policy *policy, char *buf)              \
 575 {                                                       \
 576         return sprintf(buf, "%u\n", policy->object);    \
 577 }
 578
 579 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 580 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 581 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 582 show_one(scaling_min_freq, min);
 583 show_one(scaling_max_freq, max);
 584
 585 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 586 {
 587         ssize_t ret;
 588
 589         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 590                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 591         else
 592                 ret = sprintf(buf, "%u\n", policy->cur);
 593         return ret;
 594 }
 595
 596 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 597                                 struct cpufreq_policy *new_policy);
 598
 599 /**
 600  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 601  */
 602 #define store_one(file_name, object)                    \
 603 static ssize_t store_##file_name                                        \
 604 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 605 {                                                                       \
 606         int ret, temp;                                                  \
 607         struct cpufreq_policy new_policy;                               \
 608                                                                         \
 609         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 610         if (ret)                                                        \
 611                 return -EINVAL;                                         \
 612                                                                         \
 613         ret = sscanf(buf, "%u", &new_policy.object);                    \
 614         if (ret != 1)                                                   \
 615                 return -EINVAL;                                         \
 616                                                                         \
 617         temp = new_policy.object;                                       \
 618         ret = cpufreq_set_policy(policy, &new_policy);          \
 619         if (!ret)                                                       \
 620                 policy->user_policy.object = temp;                      \
 621                                                                         \
 622         return ret ? ret : count;                                       \
 623 }
 624
 625 store_one(scaling_min_freq, min);
 626 store_one(scaling_max_freq, max);
 627
 628 /**
 629  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 630  */
 631 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 632                                         char *buf)
 633 {
 634         unsigned int cur_freq = __cpufreq_get(policy);
 635         if (!cur_freq)
 636                 return sprintf(buf, "<unknown>");
 637         return sprintf(buf, "%u\n", cur_freq);
 638 }
 639
 640 /**
 641  * show_scaling_governor - show the current policy for the specified CPU
 642  */
 643 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 644 {
 645         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 646                 return sprintf(buf, "powersave\n");
 647         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 648                 return sprintf(buf, "performance\n");
 649         else if (policy->governor)
 650                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 651                                 policy->governor->name);
 652         return -EINVAL;
 653 }
 654
 655 /**
 656  * store_scaling_governor - store policy for the specified CPU
 657  */
 658 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 659                                         const char *buf, size_t count)
 660 {
 661         int ret;
 662         char    str_governor[16];
 663         struct cpufreq_policy new_policy;
 664
 665         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 666         if (ret)
 667                 return ret;
 668
 669         ret = sscanf(buf, "%15s", str_governor);
 670         if (ret != 1)
 671                 return -EINVAL;
 672
 673         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 674                                                 &new_policy.governor))
 675                 return -EINVAL;
 676
 677         ret = cpufreq_set_policy(policy, &new_policy);
 678
 679         policy->user_policy.policy = policy->policy;
 680         policy->user_policy.governor = policy->governor;
 681
 682         if (ret)
 683                 return ret;
 684         else
 685                 return count;
 686 }
 687
 688 /**
 689  * show_scaling_driver - show the cpufreq driver currently loaded
 690  */
 691 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 692 {
 693         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 694 }
 695
 696 /**
 697  * show_scaling_available_governors - show the available CPUfreq governors
 698  */
 699 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 700                                                 char *buf)
 701 {
 702         ssize_t i = 0;
 703         struct cpufreq_governor *t;
 704
 705         if (!has_target()) {
 706                 i += sprintf(buf, "performance powersave");
 707                 goto out;
 708         }
 709
 710         for_each_governor(t) {
 711                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 712                     - (CPUFREQ_NAME_LEN + 2)))
 713                         goto out;
 714                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 715         }
 716 out:
 717         i += sprintf(&buf[i], "\n");
 718         return i;
 719 }
 720
 721 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 722 {
 723         ssize_t i = 0;
 724         unsigned int cpu;
 725
 726         for_each_cpu(cpu, mask) {
 727                 if (i)
 728                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 729                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 730                 if (i >= (PAGE_SIZE - 5))
 731                         break;
 732         }
 733         i += sprintf(&buf[i], "\n");
 734         return i;
 735 }
 736 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 737
 738 /**
 739  * show_related_cpus - show the CPUs affected by each transition even if
 740  * hw coordination is in use
 741  */
 742 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 743 {
 744         return cpufreq_show_cpus(policy->related_cpus, buf);
 745 }
 746
 747 /**
 748  * show_affected_cpus - show the CPUs affected by each transition
 749  */
 750 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 751 {
 752         return cpufreq_show_cpus(policy->cpus, buf);
 753 }
 754
 755 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 756                                         const char *buf, size_t count)
 757 {
 758         unsigned int freq = 0;
 759         unsigned int ret;
 760
 761         if (!policy->governor || !policy->governor->store_setspeed)
 762                 return -EINVAL;
 763
 764         ret = sscanf(buf, "%u", &freq);
 765         if (ret != 1)
 766                 return -EINVAL;
 767
 768         policy->governor->store_setspeed(policy, freq);
 769
 770         return count;
 771 }
 772
 773 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 774 {
 775         if (!policy->governor || !policy->governor->show_setspeed)
 776                 return sprintf(buf, "<unsupported>\n");
 777
 778         return policy->governor->show_setspeed(policy, buf);
 779 }
 780
 781 /**
 782  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 783  */
 784 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 785 {
 786         unsigned int limit;
 787         int ret;
 788         if (cpufreq_driver->bios_limit) {
 789                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 790                 if (!ret)
 791                         return sprintf(buf, "%u\n", limit);
 792         }
 793         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 794 }
 795
 796 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 797 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 798 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 799 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 800 cpufreq_freq_attr_ro(scaling_available_governors);
 801 cpufreq_freq_attr_ro(scaling_driver);
 802 cpufreq_freq_attr_ro(scaling_cur_freq);
 803 cpufreq_freq_attr_ro(bios_limit);
 804 cpufreq_freq_attr_ro(related_cpus);
 805 cpufreq_freq_attr_ro(affected_cpus);
 806 cpufreq_freq_attr_rw(scaling_min_freq);
 807 cpufreq_freq_attr_rw(scaling_max_freq);
 808 cpufreq_freq_attr_rw(scaling_governor);
 809 cpufreq_freq_attr_rw(scaling_setspeed);
 810
 811 static struct attribute *default_attrs[] = {
 812         &cpuinfo_min_freq.attr,
 813         &cpuinfo_max_freq.attr,
 814         &cpuinfo_transition_latency.attr,
 815         &scaling_min_freq.attr,
 816         &scaling_max_freq.attr,
 817         &affected_cpus.attr,
 818         &related_cpus.attr,
 819         &scaling_governor.attr,
 820         &scaling_driver.attr,
 821         &scaling_available_governors.attr,
 822         &scaling_setspeed.attr,
 823         NULL
 824 };
 825
 826 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 827 #define to_attr(a) container_of(a, struct freq_attr, attr)
 828
 829 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 830 {
 831         struct cpufreq_policy *policy = to_policy(kobj);
 832         struct freq_attr *fattr = to_attr(attr);
 833         ssize_t ret;
 834
 835         down_read(&policy->rwsem);
 836
 837         if (fattr->show)
 838                 ret = fattr->show(policy, buf);
 839         else
 840                 ret = -EIO;
 841
 842         up_read(&policy->rwsem);
 843
 844         return ret;
 845 }
 846
 847 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 848                      const char *buf, size_t count)
 849 {
 850         struct cpufreq_policy *policy = to_policy(kobj);
 851         struct freq_attr *fattr = to_attr(attr);
 852         ssize_t ret = -EINVAL;
 853
 854         get_online_cpus();
 855
 856         if (!cpu_online(policy->cpu))
 857                 goto unlock;
 858
 859         down_write(&policy->rwsem);
 860
 861         /* Updating inactive policies is invalid, so avoid doing that. */
 862         if (unlikely(policy_is_inactive(policy))) {
 863                 ret = -EBUSY;
 864                 goto unlock_policy_rwsem;
 865         }
 866
 867         if (fattr->store)
 868                 ret = fattr->store(policy, buf, count);
 869         else
 870                 ret = -EIO;
 871
 872 unlock_policy_rwsem:
 873         up_write(&policy->rwsem);
 874 unlock:
 875         put_online_cpus();
 876
 877         return ret;
 878 }
 879
 880 static void cpufreq_sysfs_release(struct kobject *kobj)
 881 {
 882         struct cpufreq_policy *policy = to_policy(kobj);
 883         pr_debug("last reference is dropped\n");
 884         complete(&policy->kobj_unregister);
 885 }
 886
 887 static const struct sysfs_ops sysfs_ops = {
 888         .show   = show,
 889         .store  = store,
 890 };
 891
 892 static struct kobj_type ktype_cpufreq = {
 893         .sysfs_ops      = &sysfs_ops,
 894         .default_attrs  = default_attrs,
 895         .release        = cpufreq_sysfs_release,
 896 };
 897
 898 struct kobject *cpufreq_global_kobject;
 899 EXPORT_SYMBOL(cpufreq_global_kobject);
 900
 901 static int cpufreq_global_kobject_usage;
 902
 903 int cpufreq_get_global_kobject(void)
 904 {
 905         if (!cpufreq_global_kobject_usage++)
 906                 return kobject_add(cpufreq_global_kobject,
 907                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 908
 909         return 0;
 910 }
 911 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 912
 913 void cpufreq_put_global_kobject(void)
 914 {
 915         if (!--cpufreq_global_kobject_usage)
 916                 kobject_del(cpufreq_global_kobject);
 917 }
 918 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 919
 920 int cpufreq_sysfs_create_file(const struct attribute *attr)
 921 {
 922         int ret = cpufreq_get_global_kobject();
 923
 924         if (!ret) {
 925                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 926                 if (ret)
 927                         cpufreq_put_global_kobject();
 928         }
 929
 930         return ret;
 931 }
 932 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 933
 934 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 935 {
 936         sysfs_remove_file(cpufreq_global_kobject, attr);
 937         cpufreq_put_global_kobject();
 938 }
 939 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 940
 941 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 942 {
 943         struct device *cpu_dev;
 944
 945         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 946
 947         if (!policy)
 948                 return 0;
 949
 950         cpu_dev = get_cpu_device(cpu);
 951         if (WARN_ON(!cpu_dev))
 952                 return 0;
 953
 954         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 955 }
 956
 957 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 958 {
 959         struct device *cpu_dev;
 960
 961         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 962
 963         cpu_dev = get_cpu_device(cpu);
 964         if (WARN_ON(!cpu_dev))
 965                 return;
 966
 967         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 968 }
 969
 970 /* Add/remove symlinks for all related CPUs */
 971 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 972 {
 973         unsigned int j;
 974         int ret = 0;
 975
 976         /* Some related CPUs might not be present (physically hotplugged) */
 977         for_each_cpu_and(j, policy->related_cpus, cpu_present_mask) {
 978                 if (j == policy->kobj_cpu)
 979                         continue;
 980
 981                 ret = add_cpu_dev_symlink(policy, j);
 982                 if (ret)
 983                         break;
 984         }
 985
 986         return ret;
 987 }
 988
 989 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
 990 {
 991         unsigned int j;
 992
 993         /* Some related CPUs might not be present (physically hotplugged) */
 994         for_each_cpu_and(j, policy->related_cpus, cpu_present_mask) {
 995                 if (j == policy->kobj_cpu)
 996                         continue;
 997
 998                 remove_cpu_dev_symlink(policy, j);
 999         }
1000 }
1001
1002 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
1003                                      struct device *dev)
1004 {
1005         struct freq_attr **drv_attr;
1006         int ret = 0;
1007
1008         /* set up files for this cpu device */
1009         drv_attr = cpufreq_driver->attr;
1010         while (drv_attr && *drv_attr) {
1011                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1012                 if (ret)
1013                         return ret;
1014                 drv_attr++;
1015         }
1016         if (cpufreq_driver->get) {
1017                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1018                 if (ret)
1019                         return ret;
1020         }
1021
1022         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1023         if (ret)
1024                 return ret;
1025
1026         if (cpufreq_driver->bios_limit) {
1027                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1028                 if (ret)
1029                         return ret;
1030         }
1031
1032         return cpufreq_add_dev_symlink(policy);
1033 }
1034
1035 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1036 {
1037         struct cpufreq_governor *gov = NULL;
1038         struct cpufreq_policy new_policy;
1039
1040         memcpy(&new_policy, policy, sizeof(*policy));
1041
1042         /* Update governor of new_policy to the governor used before hotplug */
1043         gov = find_governor(policy->last_governor);
1044         if (gov)
1045                 pr_debug("Restoring governor %s for cpu %d\n",
1046                                 policy->governor->name, policy->cpu);
1047         else
1048                 gov = CPUFREQ_DEFAULT_GOVERNOR;
1049
1050         new_policy.governor = gov;
1051
1052         /* Use the default policy if its valid. */
1053         if (cpufreq_driver->setpolicy)
1054                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
1055
1056         /* set default policy */
1057         return cpufreq_set_policy(policy, &new_policy);
1058 }
1059
1060 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
1061                                   unsigned int cpu, struct device *dev)
1062 {
1063         int ret = 0;
1064
1065         /* Has this CPU been taken care of already? */
1066         if (cpumask_test_cpu(cpu, policy->cpus))
1067                 return 0;
1068
1069         if (has_target()) {
1070                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1071                 if (ret) {
1072                         pr_err("%s: Failed to stop governor\n", __func__);
1073                         return ret;
1074                 }
1075         }
1076
1077         down_write(&policy->rwsem);
1078         cpumask_set_cpu(cpu, policy->cpus);
1079         up_write(&policy->rwsem);
1080
1081         if (has_target()) {
1082                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1083                 if (!ret)
1084                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1085
1086                 if (ret) {
1087                         pr_err("%s: Failed to start governor\n", __func__);
1088                         return ret;
1089                 }
1090         }
1091
1092         return 0;
1093 }
1094
1095 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1096 {
1097         struct cpufreq_policy *policy;
1098         unsigned long flags;
1099
1100         read_lock_irqsave(&cpufreq_driver_lock, flags);
1101         policy = per_cpu(cpufreq_cpu_data, cpu);
1102         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1103
1104         if (likely(policy)) {
1105                 /* Policy should be inactive here */
1106                 WARN_ON(!policy_is_inactive(policy));
1107
1108                 down_write(&policy->rwsem);
1109                 policy->cpu = cpu;
1110                 policy->governor = NULL;
1111                 up_write(&policy->rwsem);
1112         }
1113
1114         return policy;
1115 }
1116
1117 static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev)
1118 {
1119         struct cpufreq_policy *policy;
1120         int ret;
1121
1122         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1123         if (!policy)
1124                 return NULL;
1125
1126         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1127                 goto err_free_policy;
1128
1129         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1130                 goto err_free_cpumask;
1131
1132         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &dev->kobj,
1133                                    "cpufreq");
1134         if (ret) {
1135                 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
1136                 goto err_free_rcpumask;
1137         }
1138
1139         INIT_LIST_HEAD(&policy->policy_list);
1140         init_rwsem(&policy->rwsem);
1141         spin_lock_init(&policy->transition_lock);
1142         init_waitqueue_head(&policy->transition_wait);
1143         init_completion(&policy->kobj_unregister);
1144         INIT_WORK(&policy->update, handle_update);
1145
1146         policy->cpu = dev->id;
1147
1148         /* Set this once on allocation */
1149         policy->kobj_cpu = dev->id;
1150
1151         return policy;
1152
1153 err_free_rcpumask:
1154         free_cpumask_var(policy->related_cpus);
1155 err_free_cpumask:
1156         free_cpumask_var(policy->cpus);
1157 err_free_policy:
1158         kfree(policy);
1159
1160         return NULL;
1161 }
1162
1163 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1164 {
1165         struct kobject *kobj;
1166         struct completion *cmp;
1167
1168         if (notify)
1169                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1170                                              CPUFREQ_REMOVE_POLICY, policy);
1171
1172         down_write(&policy->rwsem);
1173         cpufreq_remove_dev_symlink(policy);
1174         kobj = &policy->kobj;
1175         cmp = &policy->kobj_unregister;
1176         up_write(&policy->rwsem);
1177         kobject_put(kobj);
1178
1179         /*
1180          * We need to make sure that the underlying kobj is
1181          * actually not referenced anymore by anybody before we
1182          * proceed with unloading.
1183          */
1184         pr_debug("waiting for dropping of refcount\n");
1185         wait_for_completion(cmp);
1186         pr_debug("wait complete\n");
1187 }
1188
1189 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1190 {
1191         unsigned long flags;
1192         int cpu;
1193
1194         /* Remove policy from list */
1195         write_lock_irqsave(&cpufreq_driver_lock, flags);
1196         list_del(&policy->policy_list);
1197
1198         for_each_cpu(cpu, policy->related_cpus)
1199                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1200         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1201
1202         cpufreq_policy_put_kobj(policy, notify);
1203         free_cpumask_var(policy->related_cpus);
1204         free_cpumask_var(policy->cpus);
1205         kfree(policy);
1206 }
1207
1208 /**
1209  * cpufreq_add_dev - add a CPU device
1210  *
1211  * Adds the cpufreq interface for a CPU device.
1212  *
1213  * The Oracle says: try running cpufreq registration/unregistration concurrently
1214  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1215  * mess up, but more thorough testing is needed. - Mathieu
1216  */
1217 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1218 {
1219         unsigned int j, cpu = dev->id;
1220         int ret = -ENOMEM;
1221         struct cpufreq_policy *policy;
1222         unsigned long flags;
1223         bool recover_policy = !sif;
1224
1225         pr_debug("adding CPU %u\n", cpu);
1226
1227         /*
1228          * Only possible if 'cpu' wasn't physically present earlier and we are
1229          * here from subsys_interface add callback. A hotplug notifier will
1230          * follow and we will handle it like logical CPU hotplug then. For now,
1231          * just create the sysfs link.
1232          */
1233         if (cpu_is_offline(cpu))
1234                 return add_cpu_dev_symlink(per_cpu(cpufreq_cpu_data, cpu), cpu);
1235
1236         /* Check if this CPU already has a policy to manage it */
1237         policy = per_cpu(cpufreq_cpu_data, cpu);
1238         if (policy && !policy_is_inactive(policy)) {
1239                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1240                 ret = cpufreq_add_policy_cpu(policy, cpu, dev);
1241                 return ret;
1242         }
1243
1244         /*
1245          * Restore the saved policy when doing light-weight init and fall back
1246          * to the full init if that fails.
1247          */
1248         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1249         if (!policy) {
1250                 recover_policy = false;
1251                 policy = cpufreq_policy_alloc(dev);
1252                 if (!policy)
1253                         goto out_release_rwsem;
1254         }
1255
1256         cpumask_copy(policy->cpus, cpumask_of(cpu));
1257
1258         /* call driver. From then on the cpufreq must be able
1259          * to accept all calls to ->verify and ->setpolicy for this CPU
1260          */
1261         ret = cpufreq_driver->init(policy);
1262         if (ret) {
1263                 pr_debug("initialization failed\n");
1264                 goto out_free_policy;
1265         }
1266
1267         down_write(&policy->rwsem);
1268
1269         /* related cpus should atleast have policy->cpus */
1270         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1271
1272         /*
1273          * affected cpus must always be the one, which are online. We aren't
1274          * managing offline cpus here.
1275          */
1276         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1277
1278         if (!recover_policy) {
1279                 policy->user_policy.min = policy->min;
1280                 policy->user_policy.max = policy->max;
1281
1282                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1283                 for_each_cpu(j, policy->related_cpus)
1284                         per_cpu(cpufreq_cpu_data, j) = policy;
1285                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1286         }
1287
1288         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1289                 policy->cur = cpufreq_driver->get(policy->cpu);
1290                 if (!policy->cur) {
1291                         pr_err("%s: ->get() failed\n", __func__);
1292                         goto out_exit_policy;
1293                 }
1294         }
1295
1296         /*
1297          * Sometimes boot loaders set CPU frequency to a value outside of
1298          * frequency table present with cpufreq core. In such cases CPU might be
1299          * unstable if it has to run on that frequency for long duration of time
1300          * and so its better to set it to a frequency which is specified in
1301          * freq-table. This also makes cpufreq stats inconsistent as
1302          * cpufreq-stats would fail to register because current frequency of CPU
1303          * isn't found in freq-table.
1304          *
1305          * Because we don't want this change to effect boot process badly, we go
1306          * for the next freq which is >= policy->cur ('cur' must be set by now,
1307          * otherwise we will end up setting freq to lowest of the table as 'cur'
1308          * is initialized to zero).
1309          *
1310          * We are passing target-freq as "policy->cur - 1" otherwise
1311          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1312          * equal to target-freq.
1313          */
1314         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1315             && has_target()) {
1316                 /* Are we running at unknown frequency ? */
1317                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1318                 if (ret == -EINVAL) {
1319                         /* Warn user and fix it */
1320                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1321                                 __func__, policy->cpu, policy->cur);
1322                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1323                                 CPUFREQ_RELATION_L);
1324
1325                         /*
1326                          * Reaching here after boot in a few seconds may not
1327                          * mean that system will remain stable at "unknown"
1328                          * frequency for longer duration. Hence, a BUG_ON().
1329                          */
1330                         BUG_ON(ret);
1331                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1332                                 __func__, policy->cpu, policy->cur);
1333                 }
1334         }
1335
1336         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1337                                      CPUFREQ_START, policy);
1338
1339         if (!recover_policy) {
1340                 ret = cpufreq_add_dev_interface(policy, dev);
1341                 if (ret)
1342                         goto out_exit_policy;
1343                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1344                                 CPUFREQ_CREATE_POLICY, policy);
1345
1346                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1347                 list_add(&policy->policy_list, &cpufreq_policy_list);
1348                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1349         }
1350
1351         ret = cpufreq_init_policy(policy);
1352         if (ret) {
1353                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1354                        __func__, cpu, ret);
1355                 goto out_remove_policy_notify;
1356         }
1357
1358         if (!recover_policy) {
1359                 policy->user_policy.policy = policy->policy;
1360                 policy->user_policy.governor = policy->governor;
1361         }
1362         up_write(&policy->rwsem);
1363
1364         kobject_uevent(&policy->kobj, KOBJ_ADD);
1365
1366         /* Callback for handling stuff after policy is ready */
1367         if (cpufreq_driver->ready)
1368                 cpufreq_driver->ready(policy);
1369
1370         pr_debug("initialization complete\n");
1371
1372         return 0;
1373
1374 out_remove_policy_notify:
1375         /* cpufreq_policy_free() will notify based on this */
1376         recover_policy = true;
1377 out_exit_policy:
1378         up_write(&policy->rwsem);
1379
1380         if (cpufreq_driver->exit)
1381                 cpufreq_driver->exit(policy);
1382 out_free_policy:
1383         cpufreq_policy_free(policy, recover_policy);
1384 out_release_rwsem:
1385         return ret;
1386 }
1387
1388 static int __cpufreq_remove_dev_prepare(struct device *dev,
1389                                         struct subsys_interface *sif)
1390 {
1391         unsigned int cpu = dev->id;
1392         int ret = 0;
1393         struct cpufreq_policy *policy;
1394
1395         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1396
1397         policy = cpufreq_cpu_get_raw(cpu);
1398         if (!policy) {
1399                 pr_debug("%s: No cpu_data found\n", __func__);
1400                 return -EINVAL;
1401         }
1402
1403         if (has_target()) {
1404                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1405                 if (ret) {
1406                         pr_err("%s: Failed to stop governor\n", __func__);
1407                         return ret;
1408                 }
1409         }
1410
1411         down_write(&policy->rwsem);
1412         cpumask_clear_cpu(cpu, policy->cpus);
1413
1414         if (policy_is_inactive(policy)) {
1415                 if (has_target())
1416                         strncpy(policy->last_governor, policy->governor->name,
1417                                 CPUFREQ_NAME_LEN);
1418         } else if (cpu == policy->cpu) {
1419                 /* Nominate new CPU */
1420                 policy->cpu = cpumask_any(policy->cpus);
1421         }
1422         up_write(&policy->rwsem);
1423
1424         /* Start governor again for active policy */
1425         if (!policy_is_inactive(policy)) {
1426                 if (has_target()) {
1427                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1428                         if (!ret)
1429                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1430
1431                         if (ret)
1432                                 pr_err("%s: Failed to start governor\n", __func__);
1433                 }
1434         } else if (cpufreq_driver->stop_cpu) {
1435                 cpufreq_driver->stop_cpu(policy);
1436         }
1437
1438         return ret;
1439 }
1440
1441 static int __cpufreq_remove_dev_finish(struct device *dev,
1442                                        struct subsys_interface *sif)
1443 {
1444         unsigned int cpu = dev->id;
1445         int ret;
1446         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1447
1448         if (!policy) {
1449                 pr_debug("%s: No cpu_data found\n", __func__);
1450                 return -EINVAL;
1451         }
1452
1453         /* Only proceed for inactive policies */
1454         if (!policy_is_inactive(policy))
1455                 return 0;
1456
1457         /* If cpu is last user of policy, free policy */
1458         if (has_target()) {
1459                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1460                 if (ret) {
1461                         pr_err("%s: Failed to exit governor\n", __func__);
1462                         return ret;
1463                 }
1464         }
1465
1466         /*
1467          * Perform the ->exit() even during light-weight tear-down,
1468          * since this is a core component, and is essential for the
1469          * subsequent light-weight ->init() to succeed.
1470          */
1471         if (cpufreq_driver->exit)
1472                 cpufreq_driver->exit(policy);
1473
1474         /* Free the policy only if the driver is getting removed. */
1475         if (sif)
1476                 cpufreq_policy_free(policy, true);
1477
1478         return 0;
1479 }
1480
1481 /**
1482  * cpufreq_remove_dev - remove a CPU device
1483  *
1484  * Removes the cpufreq interface for a CPU device.
1485  */
1486 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1487 {
1488         unsigned int cpu = dev->id;
1489         int ret;
1490
1491         /*
1492          * Only possible if 'cpu' is getting physically removed now. A hotplug
1493          * notifier should have already been called and we just need to remove
1494          * link or free policy here.
1495          */
1496         if (cpu_is_offline(cpu)) {
1497                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1498                 struct cpumask mask;
1499
1500                 if (!policy)
1501                         return 0;
1502
1503                 cpumask_copy(&mask, policy->related_cpus);
1504                 cpumask_clear_cpu(cpu, &mask);
1505
1506                 /*
1507                  * Free policy only if all policy->related_cpus are removed
1508                  * physically.
1509                  */
1510                 if (cpumask_intersects(&mask, cpu_present_mask)) {
1511                         remove_cpu_dev_symlink(policy, cpu);
1512                         return 0;
1513                 }
1514
1515                 cpufreq_policy_free(policy, true);
1516                 return 0;
1517         }
1518
1519         ret = __cpufreq_remove_dev_prepare(dev, sif);
1520
1521         if (!ret)
1522                 ret = __cpufreq_remove_dev_finish(dev, sif);
1523
1524         return ret;
1525 }
1526
1527 static void handle_update(struct work_struct *work)
1528 {
1529         struct cpufreq_policy *policy =
1530                 container_of(work, struct cpufreq_policy, update);
1531         unsigned int cpu = policy->cpu;
1532         pr_debug("handle_update for cpu %u called\n", cpu);
1533         cpufreq_update_policy(cpu);
1534 }
1535
1536 /**
1537  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1538  *      in deep trouble.
1539  *      @policy: policy managing CPUs
1540  *      @new_freq: CPU frequency the CPU actually runs at
1541  *
1542  *      We adjust to current frequency first, and need to clean up later.
1543  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1544  */
1545 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1546                                 unsigned int new_freq)
1547 {
1548         struct cpufreq_freqs freqs;
1549
1550         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1551                  policy->cur, new_freq);
1552
1553         freqs.old = policy->cur;
1554         freqs.new = new_freq;
1555
1556         cpufreq_freq_transition_begin(policy, &freqs);
1557         cpufreq_freq_transition_end(policy, &freqs, 0);
1558 }
1559
1560 /**
1561  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1562  * @cpu: CPU number
1563  *
1564  * This is the last known freq, without actually getting it from the driver.
1565  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1566  */
1567 unsigned int cpufreq_quick_get(unsigned int cpu)
1568 {
1569         struct cpufreq_policy *policy;
1570         unsigned int ret_freq = 0;
1571
1572         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1573                 return cpufreq_driver->get(cpu);
1574
1575         policy = cpufreq_cpu_get(cpu);
1576         if (policy) {
1577                 ret_freq = policy->cur;
1578                 cpufreq_cpu_put(policy);
1579         }
1580
1581         return ret_freq;
1582 }
1583 EXPORT_SYMBOL(cpufreq_quick_get);
1584
1585 /**
1586  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1587  * @cpu: CPU number
1588  *
1589  * Just return the max possible frequency for a given CPU.
1590  */
1591 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1592 {
1593         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1594         unsigned int ret_freq = 0;
1595
1596         if (policy) {
1597                 ret_freq = policy->max;
1598                 cpufreq_cpu_put(policy);
1599         }
1600
1601         return ret_freq;
1602 }
1603 EXPORT_SYMBOL(cpufreq_quick_get_max);
1604
1605 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1606 {
1607         unsigned int ret_freq = 0;
1608
1609         if (!cpufreq_driver->get)
1610                 return ret_freq;
1611
1612         ret_freq = cpufreq_driver->get(policy->cpu);
1613
1614         /* Updating inactive policies is invalid, so avoid doing that. */
1615         if (unlikely(policy_is_inactive(policy)))
1616                 return ret_freq;
1617
1618         if (ret_freq && policy->cur &&
1619                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1620                 /* verify no discrepancy between actual and
1621                                         saved value exists */
1622                 if (unlikely(ret_freq != policy->cur)) {
1623                         cpufreq_out_of_sync(policy, ret_freq);
1624                         schedule_work(&policy->update);
1625                 }
1626         }
1627
1628         return ret_freq;
1629 }
1630
1631 /**
1632  * cpufreq_get - get the current CPU frequency (in kHz)
1633  * @cpu: CPU number
1634  *
1635  * Get the CPU current (static) CPU frequency
1636  */
1637 unsigned int cpufreq_get(unsigned int cpu)
1638 {
1639         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1640         unsigned int ret_freq = 0;
1641
1642         if (policy) {
1643                 down_read(&policy->rwsem);
1644                 ret_freq = __cpufreq_get(policy);
1645                 up_read(&policy->rwsem);
1646
1647                 cpufreq_cpu_put(policy);
1648         }
1649
1650         return ret_freq;
1651 }
1652 EXPORT_SYMBOL(cpufreq_get);
1653
1654 static struct subsys_interface cpufreq_interface = {
1655         .name           = "cpufreq",
1656         .subsys         = &cpu_subsys,
1657         .add_dev        = cpufreq_add_dev,
1658         .remove_dev     = cpufreq_remove_dev,
1659 };
1660
1661 /*
1662  * In case platform wants some specific frequency to be configured
1663  * during suspend..
1664  */
1665 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1666 {
1667         int ret;
1668
1669         if (!policy->suspend_freq) {
1670                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1671                 return -EINVAL;
1672         }
1673
1674         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1675                         policy->suspend_freq);
1676
1677         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1678                         CPUFREQ_RELATION_H);
1679         if (ret)
1680                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1681                                 __func__, policy->suspend_freq, ret);
1682
1683         return ret;
1684 }
1685 EXPORT_SYMBOL(cpufreq_generic_suspend);
1686
1687 /**
1688  * cpufreq_suspend() - Suspend CPUFreq governors
1689  *
1690  * Called during system wide Suspend/Hibernate cycles for suspending governors
1691  * as some platforms can't change frequency after this point in suspend cycle.
1692  * Because some of the devices (like: i2c, regulators, etc) they use for
1693  * changing frequency are suspended quickly after this point.
1694  */
1695 void cpufreq_suspend(void)
1696 {
1697         struct cpufreq_policy *policy;
1698
1699         if (!cpufreq_driver)
1700                 return;
1701
1702         if (!has_target())
1703                 goto suspend;
1704
1705         pr_debug("%s: Suspending Governors\n", __func__);
1706
1707         for_each_active_policy(policy) {
1708                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1709                         pr_err("%s: Failed to stop governor for policy: %p\n",
1710                                 __func__, policy);
1711                 else if (cpufreq_driver->suspend
1712                     && cpufreq_driver->suspend(policy))
1713                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1714                                 policy);
1715         }
1716
1717 suspend:
1718         cpufreq_suspended = true;
1719 }
1720
1721 /**
1722  * cpufreq_resume() - Resume CPUFreq governors
1723  *
1724  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1725  * are suspended with cpufreq_suspend().
1726  */
1727 void cpufreq_resume(void)
1728 {
1729         struct cpufreq_policy *policy;
1730
1731         if (!cpufreq_driver)
1732                 return;
1733
1734         cpufreq_suspended = false;
1735
1736         if (!has_target())
1737                 return;
1738
1739         pr_debug("%s: Resuming Governors\n", __func__);
1740
1741         for_each_active_policy(policy) {
1742                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1743                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1744                                 policy);
1745                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1746                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1747                         pr_err("%s: Failed to start governor for policy: %p\n",
1748                                 __func__, policy);
1749         }
1750
1751         /*
1752          * schedule call cpufreq_update_policy() for first-online CPU, as that
1753          * wouldn't be hotplugged-out on suspend. It will verify that the
1754          * current freq is in sync with what we believe it to be.
1755          */
1756         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1757         if (WARN_ON(!policy))
1758                 return;
1759
1760         schedule_work(&policy->update);
1761 }
1762
1763 /**
1764  *      cpufreq_get_current_driver - return current driver's name
1765  *
1766  *      Return the name string of the currently loaded cpufreq driver
1767  *      or NULL, if none.
1768  */
1769 const char *cpufreq_get_current_driver(void)
1770 {
1771         if (cpufreq_driver)
1772                 return cpufreq_driver->name;
1773
1774         return NULL;
1775 }
1776 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1777
1778 /**
1779  *      cpufreq_get_driver_data - return current driver data
1780  *
1781  *      Return the private data of the currently loaded cpufreq
1782  *      driver, or NULL if no cpufreq driver is loaded.
1783  */
1784 void *cpufreq_get_driver_data(void)
1785 {
1786         if (cpufreq_driver)
1787                 return cpufreq_driver->driver_data;
1788
1789         return NULL;
1790 }
1791 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1792
1793 /*********************************************************************
1794  *                     NOTIFIER LISTS INTERFACE                      *
1795  *********************************************************************/
1796
1797 /**
1798  *      cpufreq_register_notifier - register a driver with cpufreq
1799  *      @nb: notifier function to register
1800  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1801  *
1802  *      Add a driver to one of two lists: either a list of drivers that
1803  *      are notified about clock rate changes (once before and once after
1804  *      the transition), or a list of drivers that are notified about
1805  *      changes in cpufreq policy.
1806  *
1807  *      This function may sleep, and has the same return conditions as
1808  *      blocking_notifier_chain_register.
1809  */
1810 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1811 {
1812         int ret;
1813
1814         if (cpufreq_disabled())
1815                 return -EINVAL;
1816
1817         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1818
1819         switch (list) {
1820         case CPUFREQ_TRANSITION_NOTIFIER:
1821                 ret = srcu_notifier_chain_register(
1822                                 &cpufreq_transition_notifier_list, nb);
1823                 break;
1824         case CPUFREQ_POLICY_NOTIFIER:
1825                 ret = blocking_notifier_chain_register(
1826                                 &cpufreq_policy_notifier_list, nb);
1827                 break;
1828         default:
1829                 ret = -EINVAL;
1830         }
1831
1832         return ret;
1833 }
1834 EXPORT_SYMBOL(cpufreq_register_notifier);
1835
1836 /**
1837  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1838  *      @nb: notifier block to be unregistered
1839  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1840  *
1841  *      Remove a driver from the CPU frequency notifier list.
1842  *
1843  *      This function may sleep, and has the same return conditions as
1844  *      blocking_notifier_chain_unregister.
1845  */
1846 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1847 {
1848         int ret;
1849
1850         if (cpufreq_disabled())
1851                 return -EINVAL;
1852
1853         switch (list) {
1854         case CPUFREQ_TRANSITION_NOTIFIER:
1855                 ret = srcu_notifier_chain_unregister(
1856                                 &cpufreq_transition_notifier_list, nb);
1857                 break;
1858         case CPUFREQ_POLICY_NOTIFIER:
1859                 ret = blocking_notifier_chain_unregister(
1860                                 &cpufreq_policy_notifier_list, nb);
1861                 break;
1862         default:
1863                 ret = -EINVAL;
1864         }
1865
1866         return ret;
1867 }
1868 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1869
1870
1871 /*********************************************************************
1872  *                              GOVERNORS                            *
1873  *********************************************************************/
1874
1875 /* Must set freqs->new to intermediate frequency */
1876 static int __target_intermediate(struct cpufreq_policy *policy,
1877                                  struct cpufreq_freqs *freqs, int index)
1878 {
1879         int ret;
1880
1881         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1882
1883         /* We don't need to switch to intermediate freq */
1884         if (!freqs->new)
1885                 return 0;
1886
1887         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1888                  __func__, policy->cpu, freqs->old, freqs->new);
1889
1890         cpufreq_freq_transition_begin(policy, freqs);
1891         ret = cpufreq_driver->target_intermediate(policy, index);
1892         cpufreq_freq_transition_end(policy, freqs, ret);
1893
1894         if (ret)
1895                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1896                        __func__, ret);
1897
1898         return ret;
1899 }
1900
1901 static int __target_index(struct cpufreq_policy *policy,
1902                           struct cpufreq_frequency_table *freq_table, int index)
1903 {
1904         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1905         unsigned int intermediate_freq = 0;
1906         int retval = -EINVAL;
1907         bool notify;
1908
1909         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1910         if (notify) {
1911                 /* Handle switching to intermediate frequency */
1912                 if (cpufreq_driver->get_intermediate) {
1913                         retval = __target_intermediate(policy, &freqs, index);
1914                         if (retval)
1915                                 return retval;
1916
1917                         intermediate_freq = freqs.new;
1918                         /* Set old freq to intermediate */
1919                         if (intermediate_freq)
1920                                 freqs.old = freqs.new;
1921                 }
1922
1923                 freqs.new = freq_table[index].frequency;
1924                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1925                          __func__, policy->cpu, freqs.old, freqs.new);
1926
1927                 cpufreq_freq_transition_begin(policy, &freqs);
1928         }
1929
1930         retval = cpufreq_driver->target_index(policy, index);
1931         if (retval)
1932                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1933                        retval);
1934
1935         if (notify) {
1936                 cpufreq_freq_transition_end(policy, &freqs, retval);
1937
1938                 /*
1939                  * Failed after setting to intermediate freq? Driver should have
1940                  * reverted back to initial frequency and so should we. Check
1941                  * here for intermediate_freq instead of get_intermediate, in
1942                  * case we haven't switched to intermediate freq at all.
1943                  */
1944                 if (unlikely(retval && intermediate_freq)) {
1945                         freqs.old = intermediate_freq;
1946                         freqs.new = policy->restore_freq;
1947                         cpufreq_freq_transition_begin(policy, &freqs);
1948                         cpufreq_freq_transition_end(policy, &freqs, 0);
1949                 }
1950         }
1951
1952         return retval;
1953 }
1954
1955 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1956                             unsigned int target_freq,
1957                             unsigned int relation)
1958 {
1959         unsigned int old_target_freq = target_freq;
1960         int retval = -EINVAL;
1961
1962         if (cpufreq_disabled())
1963                 return -ENODEV;
1964
1965         /* Make sure that target_freq is within supported range */
1966         if (target_freq > policy->max)
1967                 target_freq = policy->max;
1968         if (target_freq < policy->min)
1969                 target_freq = policy->min;
1970
1971         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1972                  policy->cpu, target_freq, relation, old_target_freq);
1973
1974         /*
1975          * This might look like a redundant call as we are checking it again
1976          * after finding index. But it is left intentionally for cases where
1977          * exactly same freq is called again and so we can save on few function
1978          * calls.
1979          */
1980         if (target_freq == policy->cur)
1981                 return 0;
1982
1983         /* Save last value to restore later on errors */
1984         policy->restore_freq = policy->cur;
1985
1986         if (cpufreq_driver->target)
1987                 retval = cpufreq_driver->target(policy, target_freq, relation);
1988         else if (cpufreq_driver->target_index) {
1989                 struct cpufreq_frequency_table *freq_table;
1990                 int index;
1991
1992                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1993                 if (unlikely(!freq_table)) {
1994                         pr_err("%s: Unable to find freq_table\n", __func__);
1995                         goto out;
1996                 }
1997
1998                 retval = cpufreq_frequency_table_target(policy, freq_table,
1999                                 target_freq, relation, &index);
2000                 if (unlikely(retval)) {
2001                         pr_err("%s: Unable to find matching freq\n", __func__);
2002                         goto out;
2003                 }
2004
2005                 if (freq_table[index].frequency == policy->cur) {
2006                         retval = 0;
2007                         goto out;
2008                 }
2009
2010                 retval = __target_index(policy, freq_table, index);
2011         }
2012
2013 out:
2014         return retval;
2015 }
2016 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2017
2018 int cpufreq_driver_target(struct cpufreq_policy *policy,
2019                           unsigned int target_freq,
2020                           unsigned int relation)
2021 {
2022         int ret = -EINVAL;
2023
2024         down_write(&policy->rwsem);
2025
2026         ret = __cpufreq_driver_target(policy, target_freq, relation);
2027
2028         up_write(&policy->rwsem);
2029
2030         return ret;
2031 }
2032 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2033
2034 static int __cpufreq_governor(struct cpufreq_policy *policy,
2035                                         unsigned int event)
2036 {
2037         int ret;
2038
2039         /* Only must be defined when default governor is known to have latency
2040            restrictions, like e.g. conservative or ondemand.
2041            That this is the case is already ensured in Kconfig
2042         */
2043 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2044         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2045 #else
2046         struct cpufreq_governor *gov = NULL;
2047 #endif
2048
2049         /* Don't start any governor operations if we are entering suspend */
2050         if (cpufreq_suspended)
2051                 return 0;
2052         /*
2053          * Governor might not be initiated here if ACPI _PPC changed
2054          * notification happened, so check it.
2055          */
2056         if (!policy->governor)
2057                 return -EINVAL;
2058
2059         if (policy->governor->max_transition_latency &&
2060             policy->cpuinfo.transition_latency >
2061             policy->governor->max_transition_latency) {
2062                 if (!gov)
2063                         return -EINVAL;
2064                 else {
2065                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2066                                 policy->governor->name, gov->name);
2067                         policy->governor = gov;
2068                 }
2069         }
2070
2071         if (event == CPUFREQ_GOV_POLICY_INIT)
2072                 if (!try_module_get(policy->governor->owner))
2073                         return -EINVAL;
2074
2075         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2076                  policy->cpu, event);
2077
2078         mutex_lock(&cpufreq_governor_lock);
2079         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2080             || (!policy->governor_enabled
2081             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2082                 mutex_unlock(&cpufreq_governor_lock);
2083                 return -EBUSY;
2084         }
2085
2086         if (event == CPUFREQ_GOV_STOP)
2087                 policy->governor_enabled = false;
2088         else if (event == CPUFREQ_GOV_START)
2089                 policy->governor_enabled = true;
2090
2091         mutex_unlock(&cpufreq_governor_lock);
2092
2093         ret = policy->governor->governor(policy, event);
2094
2095         if (!ret) {
2096                 if (event == CPUFREQ_GOV_POLICY_INIT)
2097                         policy->governor->initialized++;
2098                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2099                         policy->governor->initialized--;
2100         } else {
2101                 /* Restore original values */
2102                 mutex_lock(&cpufreq_governor_lock);
2103                 if (event == CPUFREQ_GOV_STOP)
2104                         policy->governor_enabled = true;
2105                 else if (event == CPUFREQ_GOV_START)
2106                         policy->governor_enabled = false;
2107                 mutex_unlock(&cpufreq_governor_lock);
2108         }
2109
2110         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2111                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2112                 module_put(policy->governor->owner);
2113
2114         return ret;
2115 }
2116
2117 int cpufreq_register_governor(struct cpufreq_governor *governor)
2118 {
2119         int err;
2120
2121         if (!governor)
2122                 return -EINVAL;
2123
2124         if (cpufreq_disabled())
2125                 return -ENODEV;
2126
2127         mutex_lock(&cpufreq_governor_mutex);
2128
2129         governor->initialized = 0;
2130         err = -EBUSY;
2131         if (!find_governor(governor->name)) {
2132                 err = 0;
2133                 list_add(&governor->governor_list, &cpufreq_governor_list);
2134         }
2135
2136         mutex_unlock(&cpufreq_governor_mutex);
2137         return err;
2138 }
2139 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2140
2141 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2142 {
2143         struct cpufreq_policy *policy;
2144         unsigned long flags;
2145
2146         if (!governor)
2147                 return;
2148
2149         if (cpufreq_disabled())
2150                 return;
2151
2152         /* clear last_governor for all inactive policies */
2153         read_lock_irqsave(&cpufreq_driver_lock, flags);
2154         for_each_inactive_policy(policy) {
2155                 if (!strcmp(policy->last_governor, governor->name)) {
2156                         policy->governor = NULL;
2157                         strcpy(policy->last_governor, "\0");
2158                 }
2159         }
2160         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2161
2162         mutex_lock(&cpufreq_governor_mutex);
2163         list_del(&governor->governor_list);
2164         mutex_unlock(&cpufreq_governor_mutex);
2165         return;
2166 }
2167 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2168
2169
2170 /*********************************************************************
2171  *                          POLICY INTERFACE                         *
2172  *********************************************************************/
2173
2174 /**
2175  * cpufreq_get_policy - get the current cpufreq_policy
2176  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2177  *      is written
2178  *
2179  * Reads the current cpufreq policy.
2180  */
2181 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2182 {
2183         struct cpufreq_policy *cpu_policy;
2184         if (!policy)
2185                 return -EINVAL;
2186
2187         cpu_policy = cpufreq_cpu_get(cpu);
2188         if (!cpu_policy)
2189                 return -EINVAL;
2190
2191         memcpy(policy, cpu_policy, sizeof(*policy));
2192
2193         cpufreq_cpu_put(cpu_policy);
2194         return 0;
2195 }
2196 EXPORT_SYMBOL(cpufreq_get_policy);
2197
2198 /*
2199  * policy : current policy.
2200  * new_policy: policy to be set.
2201  */
2202 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2203                                 struct cpufreq_policy *new_policy)
2204 {
2205         struct cpufreq_governor *old_gov;
2206         int ret;
2207
2208         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2209                  new_policy->cpu, new_policy->min, new_policy->max);
2210
2211         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2212
2213         if (new_policy->min > policy->max || new_policy->max < policy->min)
2214                 return -EINVAL;
2215
2216         /* verify the cpu speed can be set within this limit */
2217         ret = cpufreq_driver->verify(new_policy);
2218         if (ret)
2219                 return ret;
2220
2221         /* adjust if necessary - all reasons */
2222         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2223                         CPUFREQ_ADJUST, new_policy);
2224
2225         /* adjust if necessary - hardware incompatibility*/
2226         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2227                         CPUFREQ_INCOMPATIBLE, new_policy);
2228
2229         /*
2230          * verify the cpu speed can be set within this limit, which might be
2231          * different to the first one
2232          */
2233         ret = cpufreq_driver->verify(new_policy);
2234         if (ret)
2235                 return ret;
2236
2237         /* notification of the new policy */
2238         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2239                         CPUFREQ_NOTIFY, new_policy);
2240
2241         policy->min = new_policy->min;
2242         policy->max = new_policy->max;
2243
2244         pr_debug("new min and max freqs are %u - %u kHz\n",
2245                  policy->min, policy->max);
2246
2247         if (cpufreq_driver->setpolicy) {
2248                 policy->policy = new_policy->policy;
2249                 pr_debug("setting range\n");
2250                 return cpufreq_driver->setpolicy(new_policy);
2251         }
2252
2253         if (new_policy->governor == policy->governor)
2254                 goto out;
2255
2256         pr_debug("governor switch\n");
2257
2258         /* save old, working values */
2259         old_gov = policy->governor;
2260         /* end old governor */
2261         if (old_gov) {
2262                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2263                 if (ret) {
2264                         /* This can happen due to race with other operations */
2265                         pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2266                                  __func__, old_gov->name, ret);
2267                         return ret;
2268                 }
2269
2270                 up_write(&policy->rwsem);
2271                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2272                 down_write(&policy->rwsem);
2273
2274                 if (ret) {
2275                         pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2276                                __func__, old_gov->name, ret);
2277                         return ret;
2278                 }
2279         }
2280
2281         /* start new governor */
2282         policy->governor = new_policy->governor;
2283         ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2284         if (!ret) {
2285                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2286                 if (!ret)
2287                         goto out;
2288
2289                 up_write(&policy->rwsem);
2290                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2291                 down_write(&policy->rwsem);
2292         }
2293
2294         /* new governor failed, so re-start old one */
2295         pr_debug("starting governor %s failed\n", policy->governor->name);
2296         if (old_gov) {
2297                 policy->governor = old_gov;
2298                 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2299                         policy->governor = NULL;
2300                 else
2301                         __cpufreq_governor(policy, CPUFREQ_GOV_START);
2302         }
2303
2304         return ret;
2305
2306  out:
2307         pr_debug("governor: change or update limits\n");
2308         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2309 }
2310
2311 /**
2312  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2313  *      @cpu: CPU which shall be re-evaluated
2314  *
2315  *      Useful for policy notifiers which have different necessities
2316  *      at different times.
2317  */
2318 int cpufreq_update_policy(unsigned int cpu)
2319 {
2320         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2321         struct cpufreq_policy new_policy;
2322         int ret;
2323
2324         if (!policy)
2325                 return -ENODEV;
2326
2327         down_write(&policy->rwsem);
2328
2329         pr_debug("updating policy for CPU %u\n", cpu);
2330         memcpy(&new_policy, policy, sizeof(*policy));
2331         new_policy.min = policy->user_policy.min;
2332         new_policy.max = policy->user_policy.max;
2333         new_policy.policy = policy->user_policy.policy;
2334         new_policy.governor = policy->user_policy.governor;
2335
2336         /*
2337          * BIOS might change freq behind our back
2338          * -> ask driver for current freq and notify governors about a change
2339          */
2340         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2341                 new_policy.cur = cpufreq_driver->get(cpu);
2342                 if (WARN_ON(!new_policy.cur)) {
2343                         ret = -EIO;
2344                         goto unlock;
2345                 }
2346
2347                 if (!policy->cur) {
2348                         pr_debug("Driver did not initialize current freq\n");
2349                         policy->cur = new_policy.cur;
2350                 } else {
2351                         if (policy->cur != new_policy.cur && has_target())
2352                                 cpufreq_out_of_sync(policy, new_policy.cur);
2353                 }
2354         }
2355
2356         ret = cpufreq_set_policy(policy, &new_policy);
2357
2358 unlock:
2359         up_write(&policy->rwsem);
2360
2361         cpufreq_cpu_put(policy);
2362         return ret;
2363 }
2364 EXPORT_SYMBOL(cpufreq_update_policy);
2365
2366 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2367                                         unsigned long action, void *hcpu)
2368 {
2369         unsigned int cpu = (unsigned long)hcpu;
2370         struct device *dev;
2371
2372         dev = get_cpu_device(cpu);
2373         if (dev) {
2374                 switch (action & ~CPU_TASKS_FROZEN) {
2375                 case CPU_ONLINE:
2376                         cpufreq_add_dev(dev, NULL);
2377                         break;
2378
2379                 case CPU_DOWN_PREPARE:
2380                         __cpufreq_remove_dev_prepare(dev, NULL);
2381                         break;
2382
2383                 case CPU_POST_DEAD:
2384                         __cpufreq_remove_dev_finish(dev, NULL);
2385                         break;
2386
2387                 case CPU_DOWN_FAILED:
2388                         cpufreq_add_dev(dev, NULL);
2389                         break;
2390                 }
2391         }
2392         return NOTIFY_OK;
2393 }
2394
2395 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2396         .notifier_call = cpufreq_cpu_callback,
2397 };
2398
2399 /*********************************************************************
2400  *               BOOST                                               *
2401  *********************************************************************/
2402 static int cpufreq_boost_set_sw(int state)
2403 {
2404         struct cpufreq_frequency_table *freq_table;
2405         struct cpufreq_policy *policy;
2406         int ret = -EINVAL;
2407
2408         for_each_active_policy(policy) {
2409                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2410                 if (freq_table) {
2411                         ret = cpufreq_frequency_table_cpuinfo(policy,
2412                                                         freq_table);
2413                         if (ret) {
2414                                 pr_err("%s: Policy frequency update failed\n",
2415                                        __func__);
2416                                 break;
2417                         }
2418                         policy->user_policy.max = policy->max;
2419                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2420                 }
2421         }
2422
2423         return ret;
2424 }
2425
2426 int cpufreq_boost_trigger_state(int state)
2427 {
2428         unsigned long flags;
2429         int ret = 0;
2430
2431         if (cpufreq_driver->boost_enabled == state)
2432                 return 0;
2433
2434         write_lock_irqsave(&cpufreq_driver_lock, flags);
2435         cpufreq_driver->boost_enabled = state;
2436         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2437
2438         ret = cpufreq_driver->set_boost(state);
2439         if (ret) {
2440                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2441                 cpufreq_driver->boost_enabled = !state;
2442                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2443
2444                 pr_err("%s: Cannot %s BOOST\n",
2445                        __func__, state ? "enable" : "disable");
2446         }
2447
2448         return ret;
2449 }
2450
2451 int cpufreq_boost_supported(void)
2452 {
2453         if (likely(cpufreq_driver))
2454                 return cpufreq_driver->boost_supported;
2455
2456         return 0;
2457 }
2458 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2459
2460 int cpufreq_boost_enabled(void)
2461 {
2462         return cpufreq_driver->boost_enabled;
2463 }
2464 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2465
2466 /*********************************************************************
2467  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2468  *********************************************************************/
2469
2470 /**
2471  * cpufreq_register_driver - register a CPU Frequency driver
2472  * @driver_data: A struct cpufreq_driver containing the values#
2473  * submitted by the CPU Frequency driver.
2474  *
2475  * Registers a CPU Frequency driver to this core code. This code
2476  * returns zero on success, -EBUSY when another driver got here first
2477  * (and isn't unregistered in the meantime).
2478  *
2479  */
2480 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2481 {
2482         unsigned long flags;
2483         int ret;
2484
2485         if (cpufreq_disabled())
2486                 return -ENODEV;
2487
2488         if (!driver_data || !driver_data->verify || !driver_data->init ||
2489             !(driver_data->setpolicy || driver_data->target_index ||
2490                     driver_data->target) ||
2491              (driver_data->setpolicy && (driver_data->target_index ||
2492                     driver_data->target)) ||
2493              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2494                 return -EINVAL;
2495
2496         pr_debug("trying to register driver %s\n", driver_data->name);
2497
2498         write_lock_irqsave(&cpufreq_driver_lock, flags);
2499         if (cpufreq_driver) {
2500                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2501                 return -EEXIST;
2502         }
2503         cpufreq_driver = driver_data;
2504         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2505
2506         if (driver_data->setpolicy)
2507                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2508
2509         if (cpufreq_boost_supported()) {
2510                 /*
2511                  * Check if driver provides function to enable boost -
2512                  * if not, use cpufreq_boost_set_sw as default
2513                  */
2514                 if (!cpufreq_driver->set_boost)
2515                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2516
2517                 ret = cpufreq_sysfs_create_file(&boost.attr);
2518                 if (ret) {
2519                         pr_err("%s: cannot register global BOOST sysfs file\n",
2520                                __func__);
2521                         goto err_null_driver;
2522                 }
2523         }
2524
2525         ret = subsys_interface_register(&cpufreq_interface);
2526         if (ret)
2527                 goto err_boost_unreg;
2528
2529         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2530             list_empty(&cpufreq_policy_list)) {
2531                 /* if all ->init() calls failed, unregister */
2532                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2533                          driver_data->name);
2534                 goto err_if_unreg;
2535         }
2536
2537         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2538         pr_debug("driver %s up and running\n", driver_data->name);
2539
2540         return 0;
2541 err_if_unreg:
2542         subsys_interface_unregister(&cpufreq_interface);
2543 err_boost_unreg:
2544         if (cpufreq_boost_supported())
2545                 cpufreq_sysfs_remove_file(&boost.attr);
2546 err_null_driver:
2547         write_lock_irqsave(&cpufreq_driver_lock, flags);
2548         cpufreq_driver = NULL;
2549         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2550         return ret;
2551 }
2552 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2553
2554 /**
2555  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2556  *
2557  * Unregister the current CPUFreq driver. Only call this if you have
2558  * the right to do so, i.e. if you have succeeded in initialising before!
2559  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2560  * currently not initialised.
2561  */
2562 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2563 {
2564         unsigned long flags;
2565
2566         if (!cpufreq_driver || (driver != cpufreq_driver))
2567                 return -EINVAL;
2568
2569         pr_debug("unregistering driver %s\n", driver->name);
2570
2571         /* Protect against concurrent cpu hotplug */
2572         get_online_cpus();
2573         subsys_interface_unregister(&cpufreq_interface);
2574         if (cpufreq_boost_supported())
2575                 cpufreq_sysfs_remove_file(&boost.attr);
2576
2577         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2578
2579         write_lock_irqsave(&cpufreq_driver_lock, flags);
2580
2581         cpufreq_driver = NULL;
2582
2583         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2584         put_online_cpus();
2585
2586         return 0;
2587 }
2588 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2589
2590 /*
2591  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2592  * or mutexes when secondary CPUs are halted.
2593  */
2594 static struct syscore_ops cpufreq_syscore_ops = {
2595         .shutdown = cpufreq_suspend,
2596 };
2597
2598 static int __init cpufreq_core_init(void)
2599 {
2600         if (cpufreq_disabled())
2601                 return -ENODEV;
2602
2603         cpufreq_global_kobject = kobject_create();
2604         BUG_ON(!cpufreq_global_kobject);
2605
2606         register_syscore_ops(&cpufreq_syscore_ops);
2607
2608         return 0;
2609 }
2610 core_initcall(cpufreq_core_init);