drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 static DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 static inline bool has_target(void)
  51 {
  52         return cpufreq_driver->target_index || cpufreq_driver->target;
  53 }
  54
  55 /*
  56  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  57  * sections
  58  */
  59 static DECLARE_RWSEM(cpufreq_rwsem);
  60
  61 /* internal prototypes */
  62 static int __cpufreq_governor(struct cpufreq_policy *policy,
  63                 unsigned int event);
  64 static unsigned int __cpufreq_get(unsigned int cpu);
  65 static void handle_update(struct work_struct *work);
  66
  67 /**
  68  * Two notifier lists: the "policy" list is involved in the
  69  * validation process for a new CPU frequency policy; the
  70  * "transition" list for kernel code that needs to handle
  71  * changes to devices when the CPU clock speed changes.
  72  * The mutex locks both lists.
  73  */
  74 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  75 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  76
  77 static bool init_cpufreq_transition_notifier_list_called;
  78 static int __init init_cpufreq_transition_notifier_list(void)
  79 {
  80         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  81         init_cpufreq_transition_notifier_list_called = true;
  82         return 0;
  83 }
  84 pure_initcall(init_cpufreq_transition_notifier_list);
  85
  86 static int off __read_mostly;
  87 static int cpufreq_disabled(void)
  88 {
  89         return off;
  90 }
  91 void disable_cpufreq(void)
  92 {
  93         off = 1;
  94 }
  95 static LIST_HEAD(cpufreq_governor_list);
  96 static DEFINE_MUTEX(cpufreq_governor_mutex);
  97
  98 bool have_governor_per_policy(void)
  99 {
 100         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 101 }
 102 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 103
 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 105 {
 106         if (have_governor_per_policy())
 107                 return &policy->kobj;
 108         else
 109                 return cpufreq_global_kobject;
 110 }
 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 112
 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 114 {
 115         u64 idle_time;
 116         u64 cur_wall_time;
 117         u64 busy_time;
 118
 119         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 120
 121         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 122         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 127
 128         idle_time = cur_wall_time - busy_time;
 129         if (wall)
 130                 *wall = cputime_to_usecs(cur_wall_time);
 131
 132         return cputime_to_usecs(idle_time);
 133 }
 134
 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 136 {
 137         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 138
 139         if (idle_time == -1ULL)
 140                 return get_cpu_idle_time_jiffy(cpu, wall);
 141         else if (!io_busy)
 142                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 143
 144         return idle_time;
 145 }
 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 147
 148 /*
 149  * This is a generic cpufreq init() routine which can be used by cpufreq
 150  * drivers of SMP systems. It will do following:
 151  * - validate & show freq table passed
 152  * - set policies transition latency
 153  * - policy->cpus with all possible CPUs
 154  */
 155 int cpufreq_generic_init(struct cpufreq_policy *policy,
 156                 struct cpufreq_frequency_table *table,
 157                 unsigned int transition_latency)
 158 {
 159         int ret;
 160
 161         ret = cpufreq_table_validate_and_show(policy, table);
 162         if (ret) {
 163                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 164                 return ret;
 165         }
 166
 167         policy->cpuinfo.transition_latency = transition_latency;
 168
 169         /*
 170          * The driver only supports the SMP configuartion where all processors
 171          * share the clock and voltage and clock.
 172          */
 173         cpumask_setall(policy->cpus);
 174
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 178
 179 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 180 {
 181         struct cpufreq_policy *policy = NULL;
 182         unsigned long flags;
 183
 184         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 185                 return NULL;
 186
 187         if (!down_read_trylock(&cpufreq_rwsem))
 188                 return NULL;
 189
 190         /* get the cpufreq driver */
 191         read_lock_irqsave(&cpufreq_driver_lock, flags);
 192
 193         if (cpufreq_driver) {
 194                 /* get the CPU */
 195                 policy = per_cpu(cpufreq_cpu_data, cpu);
 196                 if (policy)
 197                         kobject_get(&policy->kobj);
 198         }
 199
 200         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 201
 202         if (!policy)
 203                 up_read(&cpufreq_rwsem);
 204
 205         return policy;
 206 }
 207 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 208
 209 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 210 {
 211         if (cpufreq_disabled())
 212                 return;
 213
 214         kobject_put(&policy->kobj);
 215         up_read(&cpufreq_rwsem);
 216 }
 217 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 218
 219 /*********************************************************************
 220  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 221  *********************************************************************/
 222
 223 /**
 224  * adjust_jiffies - adjust the system "loops_per_jiffy"
 225  *
 226  * This function alters the system "loops_per_jiffy" for the clock
 227  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 228  * systems as each CPU might be scaled differently. So, use the arch
 229  * per-CPU loops_per_jiffy value wherever possible.
 230  */
 231 #ifndef CONFIG_SMP
 232 static unsigned long l_p_j_ref;
 233 static unsigned int l_p_j_ref_freq;
 234
 235 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 236 {
 237         if (ci->flags & CPUFREQ_CONST_LOOPS)
 238                 return;
 239
 240         if (!l_p_j_ref_freq) {
 241                 l_p_j_ref = loops_per_jiffy;
 242                 l_p_j_ref_freq = ci->old;
 243                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 244                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 245         }
 246         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 247             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 248                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 249                                                                 ci->new);
 250                 pr_debug("scaling loops_per_jiffy to %lu "
 251                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 252         }
 253 }
 254 #else
 255 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 256 {
 257         return;
 258 }
 259 #endif
 260
 261 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 262                 struct cpufreq_freqs *freqs, unsigned int state)
 263 {
 264         BUG_ON(irqs_disabled());
 265
 266         if (cpufreq_disabled())
 267                 return;
 268
 269         freqs->flags = cpufreq_driver->flags;
 270         pr_debug("notification %u of frequency transition to %u kHz\n",
 271                 state, freqs->new);
 272
 273         switch (state) {
 274
 275         case CPUFREQ_PRECHANGE:
 276                 /* detect if the driver reported a value as "old frequency"
 277                  * which is not equal to what the cpufreq core thinks is
 278                  * "old frequency".
 279                  */
 280                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 281                         if ((policy) && (policy->cpu == freqs->cpu) &&
 282                             (policy->cur) && (policy->cur != freqs->old)) {
 283                                 pr_debug("Warning: CPU frequency is"
 284                                         " %u, cpufreq assumed %u kHz.\n",
 285                                         freqs->old, policy->cur);
 286                                 freqs->old = policy->cur;
 287                         }
 288                 }
 289                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 290                                 CPUFREQ_PRECHANGE, freqs);
 291                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 292                 break;
 293
 294         case CPUFREQ_POSTCHANGE:
 295                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 296                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 297                         (unsigned long)freqs->cpu);
 298                 trace_cpu_frequency(freqs->new, freqs->cpu);
 299                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 300                                 CPUFREQ_POSTCHANGE, freqs);
 301                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 302                         policy->cur = freqs->new;
 303                 break;
 304         }
 305 }
 306
 307 /**
 308  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 309  * on frequency transition.
 310  *
 311  * This function calls the transition notifiers and the "adjust_jiffies"
 312  * function. It is called twice on all CPU frequency changes that have
 313  * external effects.
 314  */
 315 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 316                 struct cpufreq_freqs *freqs, unsigned int state)
 317 {
 318         for_each_cpu(freqs->cpu, policy->cpus)
 319                 __cpufreq_notify_transition(policy, freqs, state);
 320 }
 321 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 322
 323
 324 /*********************************************************************
 325  *                          SYSFS INTERFACE                          *
 326  *********************************************************************/
 327
 328 static struct cpufreq_governor *__find_governor(const char *str_governor)
 329 {
 330         struct cpufreq_governor *t;
 331
 332         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 333                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 334                         return t;
 335
 336         return NULL;
 337 }
 338
 339 /**
 340  * cpufreq_parse_governor - parse a governor string
 341  */
 342 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 343                                 struct cpufreq_governor **governor)
 344 {
 345         int err = -EINVAL;
 346
 347         if (!cpufreq_driver)
 348                 goto out;
 349
 350         if (cpufreq_driver->setpolicy) {
 351                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 352                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 353                         err = 0;
 354                 } else if (!strnicmp(str_governor, "powersave",
 355                                                 CPUFREQ_NAME_LEN)) {
 356                         *policy = CPUFREQ_POLICY_POWERSAVE;
 357                         err = 0;
 358                 }
 359         } else if (has_target()) {
 360                 struct cpufreq_governor *t;
 361
 362                 mutex_lock(&cpufreq_governor_mutex);
 363
 364                 t = __find_governor(str_governor);
 365
 366                 if (t == NULL) {
 367                         int ret;
 368
 369                         mutex_unlock(&cpufreq_governor_mutex);
 370                         ret = request_module("cpufreq_%s", str_governor);
 371                         mutex_lock(&cpufreq_governor_mutex);
 372
 373                         if (ret == 0)
 374                                 t = __find_governor(str_governor);
 375                 }
 376
 377                 if (t != NULL) {
 378                         *governor = t;
 379                         err = 0;
 380                 }
 381
 382                 mutex_unlock(&cpufreq_governor_mutex);
 383         }
 384 out:
 385         return err;
 386 }
 387
 388 /**
 389  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 390  * print out cpufreq information
 391  *
 392  * Write out information from cpufreq_driver->policy[cpu]; object must be
 393  * "unsigned int".
 394  */
 395
 396 #define show_one(file_name, object)                     \
 397 static ssize_t show_##file_name                         \
 398 (struct cpufreq_policy *policy, char *buf)              \
 399 {                                                       \
 400         return sprintf(buf, "%u\n", policy->object);    \
 401 }
 402
 403 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 404 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 405 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 406 show_one(scaling_min_freq, min);
 407 show_one(scaling_max_freq, max);
 408 show_one(scaling_cur_freq, cur);
 409
 410 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 411                                 struct cpufreq_policy *new_policy);
 412
 413 /**
 414  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 415  */
 416 #define store_one(file_name, object)                    \
 417 static ssize_t store_##file_name                                        \
 418 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 419 {                                                                       \
 420         int ret;                                                        \
 421         struct cpufreq_policy new_policy;                               \
 422                                                                         \
 423         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 424         if (ret)                                                        \
 425                 return -EINVAL;                                         \
 426                                                                         \
 427         ret = sscanf(buf, "%u", &new_policy.object);                    \
 428         if (ret != 1)                                                   \
 429                 return -EINVAL;                                         \
 430                                                                         \
 431         ret = cpufreq_set_policy(policy, &new_policy);          \
 432         policy->user_policy.object = policy->object;                    \
 433                                                                         \
 434         return ret ? ret : count;                                       \
 435 }
 436
 437 store_one(scaling_min_freq, min);
 438 store_one(scaling_max_freq, max);
 439
 440 /**
 441  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 442  */
 443 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 444                                         char *buf)
 445 {
 446         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 447         if (!cur_freq)
 448                 return sprintf(buf, "<unknown>");
 449         return sprintf(buf, "%u\n", cur_freq);
 450 }
 451
 452 /**
 453  * show_scaling_governor - show the current policy for the specified CPU
 454  */
 455 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 456 {
 457         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 458                 return sprintf(buf, "powersave\n");
 459         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 460                 return sprintf(buf, "performance\n");
 461         else if (policy->governor)
 462                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 463                                 policy->governor->name);
 464         return -EINVAL;
 465 }
 466
 467 /**
 468  * store_scaling_governor - store policy for the specified CPU
 469  */
 470 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 471                                         const char *buf, size_t count)
 472 {
 473         int ret;
 474         char    str_governor[16];
 475         struct cpufreq_policy new_policy;
 476
 477         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 478         if (ret)
 479                 return ret;
 480
 481         ret = sscanf(buf, "%15s", str_governor);
 482         if (ret != 1)
 483                 return -EINVAL;
 484
 485         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 486                                                 &new_policy.governor))
 487                 return -EINVAL;
 488
 489         ret = cpufreq_set_policy(policy, &new_policy);
 490
 491         policy->user_policy.policy = policy->policy;
 492         policy->user_policy.governor = policy->governor;
 493
 494         if (ret)
 495                 return ret;
 496         else
 497                 return count;
 498 }
 499
 500 /**
 501  * show_scaling_driver - show the cpufreq driver currently loaded
 502  */
 503 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 504 {
 505         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 506 }
 507
 508 /**
 509  * show_scaling_available_governors - show the available CPUfreq governors
 510  */
 511 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 512                                                 char *buf)
 513 {
 514         ssize_t i = 0;
 515         struct cpufreq_governor *t;
 516
 517         if (!has_target()) {
 518                 i += sprintf(buf, "performance powersave");
 519                 goto out;
 520         }
 521
 522         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 523                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 524                     - (CPUFREQ_NAME_LEN + 2)))
 525                         goto out;
 526                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 527         }
 528 out:
 529         i += sprintf(&buf[i], "\n");
 530         return i;
 531 }
 532
 533 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 534 {
 535         ssize_t i = 0;
 536         unsigned int cpu;
 537
 538         for_each_cpu(cpu, mask) {
 539                 if (i)
 540                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 541                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 542                 if (i >= (PAGE_SIZE - 5))
 543                         break;
 544         }
 545         i += sprintf(&buf[i], "\n");
 546         return i;
 547 }
 548 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 549
 550 /**
 551  * show_related_cpus - show the CPUs affected by each transition even if
 552  * hw coordination is in use
 553  */
 554 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 555 {
 556         return cpufreq_show_cpus(policy->related_cpus, buf);
 557 }
 558
 559 /**
 560  * show_affected_cpus - show the CPUs affected by each transition
 561  */
 562 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 563 {
 564         return cpufreq_show_cpus(policy->cpus, buf);
 565 }
 566
 567 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 568                                         const char *buf, size_t count)
 569 {
 570         unsigned int freq = 0;
 571         unsigned int ret;
 572
 573         if (!policy->governor || !policy->governor->store_setspeed)
 574                 return -EINVAL;
 575
 576         ret = sscanf(buf, "%u", &freq);
 577         if (ret != 1)
 578                 return -EINVAL;
 579
 580         policy->governor->store_setspeed(policy, freq);
 581
 582         return count;
 583 }
 584
 585 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 586 {
 587         if (!policy->governor || !policy->governor->show_setspeed)
 588                 return sprintf(buf, "<unsupported>\n");
 589
 590         return policy->governor->show_setspeed(policy, buf);
 591 }
 592
 593 /**
 594  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 595  */
 596 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 597 {
 598         unsigned int limit;
 599         int ret;
 600         if (cpufreq_driver->bios_limit) {
 601                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 602                 if (!ret)
 603                         return sprintf(buf, "%u\n", limit);
 604         }
 605         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 606 }
 607
 608 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 609 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 610 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 611 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 612 cpufreq_freq_attr_ro(scaling_available_governors);
 613 cpufreq_freq_attr_ro(scaling_driver);
 614 cpufreq_freq_attr_ro(scaling_cur_freq);
 615 cpufreq_freq_attr_ro(bios_limit);
 616 cpufreq_freq_attr_ro(related_cpus);
 617 cpufreq_freq_attr_ro(affected_cpus);
 618 cpufreq_freq_attr_rw(scaling_min_freq);
 619 cpufreq_freq_attr_rw(scaling_max_freq);
 620 cpufreq_freq_attr_rw(scaling_governor);
 621 cpufreq_freq_attr_rw(scaling_setspeed);
 622
 623 static struct attribute *default_attrs[] = {
 624         &cpuinfo_min_freq.attr,
 625         &cpuinfo_max_freq.attr,
 626         &cpuinfo_transition_latency.attr,
 627         &scaling_min_freq.attr,
 628         &scaling_max_freq.attr,
 629         &affected_cpus.attr,
 630         &related_cpus.attr,
 631         &scaling_governor.attr,
 632         &scaling_driver.attr,
 633         &scaling_available_governors.attr,
 634         &scaling_setspeed.attr,
 635         NULL
 636 };
 637
 638 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 639 #define to_attr(a) container_of(a, struct freq_attr, attr)
 640
 641 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 642 {
 643         struct cpufreq_policy *policy = to_policy(kobj);
 644         struct freq_attr *fattr = to_attr(attr);
 645         ssize_t ret;
 646
 647         if (!down_read_trylock(&cpufreq_rwsem))
 648                 return -EINVAL;
 649
 650         down_read(&policy->rwsem);
 651
 652         if (fattr->show)
 653                 ret = fattr->show(policy, buf);
 654         else
 655                 ret = -EIO;
 656
 657         up_read(&policy->rwsem);
 658         up_read(&cpufreq_rwsem);
 659
 660         return ret;
 661 }
 662
 663 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 664                      const char *buf, size_t count)
 665 {
 666         struct cpufreq_policy *policy = to_policy(kobj);
 667         struct freq_attr *fattr = to_attr(attr);
 668         ssize_t ret = -EINVAL;
 669
 670         get_online_cpus();
 671
 672         if (!cpu_online(policy->cpu))
 673                 goto unlock;
 674
 675         if (!down_read_trylock(&cpufreq_rwsem))
 676                 goto unlock;
 677
 678         down_write(&policy->rwsem);
 679
 680         if (fattr->store)
 681                 ret = fattr->store(policy, buf, count);
 682         else
 683                 ret = -EIO;
 684
 685         up_write(&policy->rwsem);
 686
 687         up_read(&cpufreq_rwsem);
 688 unlock:
 689         put_online_cpus();
 690
 691         return ret;
 692 }
 693
 694 static void cpufreq_sysfs_release(struct kobject *kobj)
 695 {
 696         struct cpufreq_policy *policy = to_policy(kobj);
 697         pr_debug("last reference is dropped\n");
 698         complete(&policy->kobj_unregister);
 699 }
 700
 701 static const struct sysfs_ops sysfs_ops = {
 702         .show   = show,
 703         .store  = store,
 704 };
 705
 706 static struct kobj_type ktype_cpufreq = {
 707         .sysfs_ops      = &sysfs_ops,
 708         .default_attrs  = default_attrs,
 709         .release        = cpufreq_sysfs_release,
 710 };
 711
 712 struct kobject *cpufreq_global_kobject;
 713 EXPORT_SYMBOL(cpufreq_global_kobject);
 714
 715 static int cpufreq_global_kobject_usage;
 716
 717 int cpufreq_get_global_kobject(void)
 718 {
 719         if (!cpufreq_global_kobject_usage++)
 720                 return kobject_add(cpufreq_global_kobject,
 721                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 722
 723         return 0;
 724 }
 725 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 726
 727 void cpufreq_put_global_kobject(void)
 728 {
 729         if (!--cpufreq_global_kobject_usage)
 730                 kobject_del(cpufreq_global_kobject);
 731 }
 732 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 733
 734 int cpufreq_sysfs_create_file(const struct attribute *attr)
 735 {
 736         int ret = cpufreq_get_global_kobject();
 737
 738         if (!ret) {
 739                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 740                 if (ret)
 741                         cpufreq_put_global_kobject();
 742         }
 743
 744         return ret;
 745 }
 746 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 747
 748 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 749 {
 750         sysfs_remove_file(cpufreq_global_kobject, attr);
 751         cpufreq_put_global_kobject();
 752 }
 753 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 754
 755 /* symlink affected CPUs */
 756 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 757 {
 758         unsigned int j;
 759         int ret = 0;
 760
 761         for_each_cpu(j, policy->cpus) {
 762                 struct device *cpu_dev;
 763
 764                 if (j == policy->cpu)
 765                         continue;
 766
 767                 pr_debug("Adding link for CPU: %u\n", j);
 768                 cpu_dev = get_cpu_device(j);
 769                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 770                                         "cpufreq");
 771                 if (ret)
 772                         break;
 773         }
 774         return ret;
 775 }
 776
 777 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 778                                      struct device *dev)
 779 {
 780         struct freq_attr **drv_attr;
 781         int ret = 0;
 782
 783         /* prepare interface data */
 784         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 785                                    &dev->kobj, "cpufreq");
 786         if (ret)
 787                 return ret;
 788
 789         /* set up files for this cpu device */
 790         drv_attr = cpufreq_driver->attr;
 791         while ((drv_attr) && (*drv_attr)) {
 792                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 793                 if (ret)
 794                         goto err_out_kobj_put;
 795                 drv_attr++;
 796         }
 797         if (cpufreq_driver->get) {
 798                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 799                 if (ret)
 800                         goto err_out_kobj_put;
 801         }
 802         if (has_target()) {
 803                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 804                 if (ret)
 805                         goto err_out_kobj_put;
 806         }
 807         if (cpufreq_driver->bios_limit) {
 808                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 809                 if (ret)
 810                         goto err_out_kobj_put;
 811         }
 812
 813         ret = cpufreq_add_dev_symlink(policy);
 814         if (ret)
 815                 goto err_out_kobj_put;
 816
 817         return ret;
 818
 819 err_out_kobj_put:
 820         kobject_put(&policy->kobj);
 821         wait_for_completion(&policy->kobj_unregister);
 822         return ret;
 823 }
 824
 825 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 826 {
 827         struct cpufreq_policy new_policy;
 828         int ret = 0;
 829
 830         memcpy(&new_policy, policy, sizeof(*policy));
 831
 832         /* Use the default policy if its valid. */
 833         if (cpufreq_driver->setpolicy)
 834                 cpufreq_parse_governor(policy->governor->name,
 835                                         &new_policy.policy, NULL);
 836
 837         /* assure that the starting sequence is run in cpufreq_set_policy */
 838         policy->governor = NULL;
 839
 840         /* set default policy */
 841         ret = cpufreq_set_policy(policy, &new_policy);
 842         if (ret) {
 843                 pr_debug("setting policy failed\n");
 844                 if (cpufreq_driver->exit)
 845                         cpufreq_driver->exit(policy);
 846         }
 847 }
 848
 849 #ifdef CONFIG_HOTPLUG_CPU
 850 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 851                                   unsigned int cpu, struct device *dev)
 852 {
 853         int ret = 0;
 854         unsigned long flags;
 855
 856         if (has_target()) {
 857                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 858                 if (ret) {
 859                         pr_err("%s: Failed to stop governor\n", __func__);
 860                         return ret;
 861                 }
 862         }
 863
 864         down_write(&policy->rwsem);
 865
 866         write_lock_irqsave(&cpufreq_driver_lock, flags);
 867
 868         cpumask_set_cpu(cpu, policy->cpus);
 869         per_cpu(cpufreq_cpu_data, cpu) = policy;
 870         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 871
 872         up_write(&policy->rwsem);
 873
 874         if (has_target()) {
 875                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 876                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 877                         pr_err("%s: Failed to start governor\n", __func__);
 878                         return ret;
 879                 }
 880         }
 881
 882         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 883 }
 884 #endif
 885
 886 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 887 {
 888         struct cpufreq_policy *policy;
 889         unsigned long flags;
 890
 891         read_lock_irqsave(&cpufreq_driver_lock, flags);
 892
 893         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 894
 895         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 896
 897         return policy;
 898 }
 899
 900 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 901 {
 902         struct cpufreq_policy *policy;
 903
 904         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 905         if (!policy)
 906                 return NULL;
 907
 908         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 909                 goto err_free_policy;
 910
 911         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 912                 goto err_free_cpumask;
 913
 914         INIT_LIST_HEAD(&policy->policy_list);
 915         init_rwsem(&policy->rwsem);
 916
 917         return policy;
 918
 919 err_free_cpumask:
 920         free_cpumask_var(policy->cpus);
 921 err_free_policy:
 922         kfree(policy);
 923
 924         return NULL;
 925 }
 926
 927 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
 928 {
 929         struct kobject *kobj;
 930         struct completion *cmp;
 931
 932         down_read(&policy->rwsem);
 933         kobj = &policy->kobj;
 934         cmp = &policy->kobj_unregister;
 935         up_read(&policy->rwsem);
 936         kobject_put(kobj);
 937
 938         /*
 939          * We need to make sure that the underlying kobj is
 940          * actually not referenced anymore by anybody before we
 941          * proceed with unloading.
 942          */
 943         pr_debug("waiting for dropping of refcount\n");
 944         wait_for_completion(cmp);
 945         pr_debug("wait complete\n");
 946 }
 947
 948 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 949 {
 950         free_cpumask_var(policy->related_cpus);
 951         free_cpumask_var(policy->cpus);
 952         kfree(policy);
 953 }
 954
 955 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 956 {
 957         if (WARN_ON(cpu == policy->cpu))
 958                 return;
 959
 960         down_write(&policy->rwsem);
 961
 962         policy->last_cpu = policy->cpu;
 963         policy->cpu = cpu;
 964
 965         up_write(&policy->rwsem);
 966
 967         cpufreq_frequency_table_update_policy_cpu(policy);
 968         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 969                         CPUFREQ_UPDATE_POLICY_CPU, policy);
 970 }
 971
 972 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 973                              bool frozen)
 974 {
 975         unsigned int j, cpu = dev->id;
 976         int ret = -ENOMEM;
 977         struct cpufreq_policy *policy;
 978         unsigned long flags;
 979 #ifdef CONFIG_HOTPLUG_CPU
 980         struct cpufreq_policy *tpolicy;
 981         struct cpufreq_governor *gov;
 982 #endif
 983
 984         if (cpu_is_offline(cpu))
 985                 return 0;
 986
 987         pr_debug("adding CPU %u\n", cpu);
 988
 989 #ifdef CONFIG_SMP
 990         /* check whether a different CPU already registered this
 991          * CPU because it is in the same boat. */
 992         policy = cpufreq_cpu_get(cpu);
 993         if (unlikely(policy)) {
 994                 cpufreq_cpu_put(policy);
 995                 return 0;
 996         }
 997 #endif
 998
 999         if (!down_read_trylock(&cpufreq_rwsem))
1000                 return 0;
1001
1002 #ifdef CONFIG_HOTPLUG_CPU
1003         /* Check if this cpu was hot-unplugged earlier and has siblings */
1004         read_lock_irqsave(&cpufreq_driver_lock, flags);
1005         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1006                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1007                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1008                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1009                         up_read(&cpufreq_rwsem);
1010                         return ret;
1011                 }
1012         }
1013         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1014 #endif
1015
1016         /*
1017          * Restore the saved policy when doing light-weight init and fall back
1018          * to the full init if that fails.
1019          */
1020         policy = frozen ? cpufreq_policy_restore(cpu) : NULL;
1021         if (!policy) {
1022                 frozen = false;
1023                 policy = cpufreq_policy_alloc();
1024                 if (!policy)
1025                         goto nomem_out;
1026         }
1027
1028         /*
1029          * In the resume path, since we restore a saved policy, the assignment
1030          * to policy->cpu is like an update of the existing policy, rather than
1031          * the creation of a brand new one. So we need to perform this update
1032          * by invoking update_policy_cpu().
1033          */
1034         if (frozen && cpu != policy->cpu)
1035                 update_policy_cpu(policy, cpu);
1036         else
1037                 policy->cpu = cpu;
1038
1039         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1040         cpumask_copy(policy->cpus, cpumask_of(cpu));
1041
1042         init_completion(&policy->kobj_unregister);
1043         INIT_WORK(&policy->update, handle_update);
1044
1045         /* call driver. From then on the cpufreq must be able
1046          * to accept all calls to ->verify and ->setpolicy for this CPU
1047          */
1048         ret = cpufreq_driver->init(policy);
1049         if (ret) {
1050                 pr_debug("initialization failed\n");
1051                 goto err_set_policy_cpu;
1052         }
1053
1054         if (cpufreq_driver->get) {
1055                 policy->cur = cpufreq_driver->get(policy->cpu);
1056                 if (!policy->cur) {
1057                         pr_err("%s: ->get() failed\n", __func__);
1058                         goto err_get_freq;
1059                 }
1060         }
1061
1062         /* related cpus should atleast have policy->cpus */
1063         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1064
1065         /*
1066          * affected cpus must always be the one, which are online. We aren't
1067          * managing offline cpus here.
1068          */
1069         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1070
1071         if (!frozen) {
1072                 policy->user_policy.min = policy->min;
1073                 policy->user_policy.max = policy->max;
1074         }
1075
1076         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1077                                      CPUFREQ_START, policy);
1078
1079 #ifdef CONFIG_HOTPLUG_CPU
1080         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1081         if (gov) {
1082                 policy->governor = gov;
1083                 pr_debug("Restoring governor %s for cpu %d\n",
1084                        policy->governor->name, cpu);
1085         }
1086 #endif
1087
1088         write_lock_irqsave(&cpufreq_driver_lock, flags);
1089         for_each_cpu(j, policy->cpus)
1090                 per_cpu(cpufreq_cpu_data, j) = policy;
1091         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1092
1093         if (!frozen) {
1094                 ret = cpufreq_add_dev_interface(policy, dev);
1095                 if (ret)
1096                         goto err_out_unregister;
1097         }
1098
1099         write_lock_irqsave(&cpufreq_driver_lock, flags);
1100         list_add(&policy->policy_list, &cpufreq_policy_list);
1101         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1102
1103         cpufreq_init_policy(policy);
1104
1105         if (!frozen) {
1106                 policy->user_policy.policy = policy->policy;
1107                 policy->user_policy.governor = policy->governor;
1108         }
1109
1110         kobject_uevent(&policy->kobj, KOBJ_ADD);
1111         up_read(&cpufreq_rwsem);
1112
1113         pr_debug("initialization complete\n");
1114
1115         return 0;
1116
1117 err_out_unregister:
1118         write_lock_irqsave(&cpufreq_driver_lock, flags);
1119         for_each_cpu(j, policy->cpus)
1120                 per_cpu(cpufreq_cpu_data, j) = NULL;
1121         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1122
1123 err_get_freq:
1124         if (cpufreq_driver->exit)
1125                 cpufreq_driver->exit(policy);
1126 err_set_policy_cpu:
1127         if (frozen) {
1128                 /* Do not leave stale fallback data behind. */
1129                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1130                 cpufreq_policy_put_kobj(policy);
1131         }
1132         cpufreq_policy_free(policy);
1133
1134 nomem_out:
1135         up_read(&cpufreq_rwsem);
1136
1137         return ret;
1138 }
1139
1140 /**
1141  * cpufreq_add_dev - add a CPU device
1142  *
1143  * Adds the cpufreq interface for a CPU device.
1144  *
1145  * The Oracle says: try running cpufreq registration/unregistration concurrently
1146  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1147  * mess up, but more thorough testing is needed. - Mathieu
1148  */
1149 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1150 {
1151         return __cpufreq_add_dev(dev, sif, false);
1152 }
1153
1154 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1155                                            unsigned int old_cpu)
1156 {
1157         struct device *cpu_dev;
1158         int ret;
1159
1160         /* first sibling now owns the new sysfs dir */
1161         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1162
1163         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1164         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1165         if (ret) {
1166                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1167
1168                 down_write(&policy->rwsem);
1169                 cpumask_set_cpu(old_cpu, policy->cpus);
1170                 up_write(&policy->rwsem);
1171
1172                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1173                                         "cpufreq");
1174
1175                 return -EINVAL;
1176         }
1177
1178         return cpu_dev->id;
1179 }
1180
1181 static int __cpufreq_remove_dev_prepare(struct device *dev,
1182                                         struct subsys_interface *sif,
1183                                         bool frozen)
1184 {
1185         unsigned int cpu = dev->id, cpus;
1186         int new_cpu, ret;
1187         unsigned long flags;
1188         struct cpufreq_policy *policy;
1189
1190         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1191
1192         write_lock_irqsave(&cpufreq_driver_lock, flags);
1193
1194         policy = per_cpu(cpufreq_cpu_data, cpu);
1195
1196         /* Save the policy somewhere when doing a light-weight tear-down */
1197         if (frozen)
1198                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1199
1200         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1201
1202         if (!policy) {
1203                 pr_debug("%s: No cpu_data found\n", __func__);
1204                 return -EINVAL;
1205         }
1206
1207         if (has_target()) {
1208                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1209                 if (ret) {
1210                         pr_err("%s: Failed to stop governor\n", __func__);
1211                         return ret;
1212                 }
1213         }
1214
1215 #ifdef CONFIG_HOTPLUG_CPU
1216         if (!cpufreq_driver->setpolicy)
1217                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1218                         policy->governor->name, CPUFREQ_NAME_LEN);
1219 #endif
1220
1221         down_read(&policy->rwsem);
1222         cpus = cpumask_weight(policy->cpus);
1223         up_read(&policy->rwsem);
1224
1225         if (cpu != policy->cpu) {
1226                 if (!frozen)
1227                         sysfs_remove_link(&dev->kobj, "cpufreq");
1228         } else if (cpus > 1) {
1229                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1230                 if (new_cpu >= 0) {
1231                         update_policy_cpu(policy, new_cpu);
1232
1233                         if (!frozen) {
1234                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1235                                                 __func__, new_cpu, cpu);
1236                         }
1237                 }
1238         }
1239
1240         return 0;
1241 }
1242
1243 static int __cpufreq_remove_dev_finish(struct device *dev,
1244                                        struct subsys_interface *sif,
1245                                        bool frozen)
1246 {
1247         unsigned int cpu = dev->id, cpus;
1248         int ret;
1249         unsigned long flags;
1250         struct cpufreq_policy *policy;
1251
1252         read_lock_irqsave(&cpufreq_driver_lock, flags);
1253         policy = per_cpu(cpufreq_cpu_data, cpu);
1254         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1255
1256         if (!policy) {
1257                 pr_debug("%s: No cpu_data found\n", __func__);
1258                 return -EINVAL;
1259         }
1260
1261         down_write(&policy->rwsem);
1262         cpus = cpumask_weight(policy->cpus);
1263
1264         if (cpus > 1)
1265                 cpumask_clear_cpu(cpu, policy->cpus);
1266         up_write(&policy->rwsem);
1267
1268         /* If cpu is last user of policy, free policy */
1269         if (cpus == 1) {
1270                 if (has_target()) {
1271                         ret = __cpufreq_governor(policy,
1272                                         CPUFREQ_GOV_POLICY_EXIT);
1273                         if (ret) {
1274                                 pr_err("%s: Failed to exit governor\n",
1275                                                 __func__);
1276                                 return ret;
1277                         }
1278                 }
1279
1280                 if (!frozen)
1281                         cpufreq_policy_put_kobj(policy);
1282
1283                 /*
1284                  * Perform the ->exit() even during light-weight tear-down,
1285                  * since this is a core component, and is essential for the
1286                  * subsequent light-weight ->init() to succeed.
1287                  */
1288                 if (cpufreq_driver->exit)
1289                         cpufreq_driver->exit(policy);
1290
1291                 /* Remove policy from list of active policies */
1292                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1293                 list_del(&policy->policy_list);
1294                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1295
1296                 if (!frozen)
1297                         cpufreq_policy_free(policy);
1298         } else {
1299                 if (has_target()) {
1300                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1301                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1302                                 pr_err("%s: Failed to start governor\n",
1303                                                 __func__);
1304                                 return ret;
1305                         }
1306                 }
1307         }
1308
1309         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1310         return 0;
1311 }
1312
1313 /**
1314  * cpufreq_remove_dev - remove a CPU device
1315  *
1316  * Removes the cpufreq interface for a CPU device.
1317  */
1318 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1319 {
1320         unsigned int cpu = dev->id;
1321         int ret;
1322
1323         if (cpu_is_offline(cpu))
1324                 return 0;
1325
1326         ret = __cpufreq_remove_dev_prepare(dev, sif, false);
1327
1328         if (!ret)
1329                 ret = __cpufreq_remove_dev_finish(dev, sif, false);
1330
1331         return ret;
1332 }
1333
1334 static void handle_update(struct work_struct *work)
1335 {
1336         struct cpufreq_policy *policy =
1337                 container_of(work, struct cpufreq_policy, update);
1338         unsigned int cpu = policy->cpu;
1339         pr_debug("handle_update for cpu %u called\n", cpu);
1340         cpufreq_update_policy(cpu);
1341 }
1342
1343 /**
1344  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1345  *      in deep trouble.
1346  *      @cpu: cpu number
1347  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1348  *      @new_freq: CPU frequency the CPU actually runs at
1349  *
1350  *      We adjust to current frequency first, and need to clean up later.
1351  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1352  */
1353 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1354                                 unsigned int new_freq)
1355 {
1356         struct cpufreq_policy *policy;
1357         struct cpufreq_freqs freqs;
1358         unsigned long flags;
1359
1360         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1361                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1362
1363         freqs.old = old_freq;
1364         freqs.new = new_freq;
1365
1366         read_lock_irqsave(&cpufreq_driver_lock, flags);
1367         policy = per_cpu(cpufreq_cpu_data, cpu);
1368         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1369
1370         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1371         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1372 }
1373
1374 /**
1375  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1376  * @cpu: CPU number
1377  *
1378  * This is the last known freq, without actually getting it from the driver.
1379  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1380  */
1381 unsigned int cpufreq_quick_get(unsigned int cpu)
1382 {
1383         struct cpufreq_policy *policy;
1384         unsigned int ret_freq = 0;
1385
1386         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1387                 return cpufreq_driver->get(cpu);
1388
1389         policy = cpufreq_cpu_get(cpu);
1390         if (policy) {
1391                 ret_freq = policy->cur;
1392                 cpufreq_cpu_put(policy);
1393         }
1394
1395         return ret_freq;
1396 }
1397 EXPORT_SYMBOL(cpufreq_quick_get);
1398
1399 /**
1400  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1401  * @cpu: CPU number
1402  *
1403  * Just return the max possible frequency for a given CPU.
1404  */
1405 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1406 {
1407         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1408         unsigned int ret_freq = 0;
1409
1410         if (policy) {
1411                 ret_freq = policy->max;
1412                 cpufreq_cpu_put(policy);
1413         }
1414
1415         return ret_freq;
1416 }
1417 EXPORT_SYMBOL(cpufreq_quick_get_max);
1418
1419 static unsigned int __cpufreq_get(unsigned int cpu)
1420 {
1421         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1422         unsigned int ret_freq = 0;
1423
1424         if (!cpufreq_driver->get)
1425                 return ret_freq;
1426
1427         ret_freq = cpufreq_driver->get(cpu);
1428
1429         if (ret_freq && policy->cur &&
1430                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1431                 /* verify no discrepancy between actual and
1432                                         saved value exists */
1433                 if (unlikely(ret_freq != policy->cur)) {
1434                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1435                         schedule_work(&policy->update);
1436                 }
1437         }
1438
1439         return ret_freq;
1440 }
1441
1442 /**
1443  * cpufreq_get - get the current CPU frequency (in kHz)
1444  * @cpu: CPU number
1445  *
1446  * Get the CPU current (static) CPU frequency
1447  */
1448 unsigned int cpufreq_get(unsigned int cpu)
1449 {
1450         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1451         unsigned int ret_freq = 0;
1452
1453         if (cpufreq_disabled() || !cpufreq_driver)
1454                 return -ENOENT;
1455
1456         BUG_ON(!policy);
1457
1458         if (!down_read_trylock(&cpufreq_rwsem))
1459                 return 0;
1460
1461         down_read(&policy->rwsem);
1462
1463         ret_freq = __cpufreq_get(cpu);
1464
1465         up_read(&policy->rwsem);
1466         up_read(&cpufreq_rwsem);
1467
1468         return ret_freq;
1469 }
1470 EXPORT_SYMBOL(cpufreq_get);
1471
1472 static struct subsys_interface cpufreq_interface = {
1473         .name           = "cpufreq",
1474         .subsys         = &cpu_subsys,
1475         .add_dev        = cpufreq_add_dev,
1476         .remove_dev     = cpufreq_remove_dev,
1477 };
1478
1479 /**
1480  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1481  *
1482  * This function is only executed for the boot processor.  The other CPUs
1483  * have been put offline by means of CPU hotplug.
1484  */
1485 static int cpufreq_bp_suspend(void)
1486 {
1487         int ret = 0;
1488
1489         int cpu = smp_processor_id();
1490         struct cpufreq_policy *policy;
1491
1492         pr_debug("suspending cpu %u\n", cpu);
1493
1494         /* If there's no policy for the boot CPU, we have nothing to do. */
1495         policy = cpufreq_cpu_get(cpu);
1496         if (!policy)
1497                 return 0;
1498
1499         if (cpufreq_driver->suspend) {
1500                 ret = cpufreq_driver->suspend(policy);
1501                 if (ret)
1502                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1503                                         "step on CPU %u\n", policy->cpu);
1504         }
1505
1506         cpufreq_cpu_put(policy);
1507         return ret;
1508 }
1509
1510 /**
1511  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1512  *
1513  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1514  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1515  *          restored. It will verify that the current freq is in sync with
1516  *          what we believe it to be. This is a bit later than when it
1517  *          should be, but nonethteless it's better than calling
1518  *          cpufreq_driver->get() here which might re-enable interrupts...
1519  *
1520  * This function is only executed for the boot CPU.  The other CPUs have not
1521  * been turned on yet.
1522  */
1523 static void cpufreq_bp_resume(void)
1524 {
1525         int ret = 0;
1526
1527         int cpu = smp_processor_id();
1528         struct cpufreq_policy *policy;
1529
1530         pr_debug("resuming cpu %u\n", cpu);
1531
1532         /* If there's no policy for the boot CPU, we have nothing to do. */
1533         policy = cpufreq_cpu_get(cpu);
1534         if (!policy)
1535                 return;
1536
1537         if (cpufreq_driver->resume) {
1538                 ret = cpufreq_driver->resume(policy);
1539                 if (ret) {
1540                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1541                                         "step on CPU %u\n", policy->cpu);
1542                         goto fail;
1543                 }
1544         }
1545
1546         schedule_work(&policy->update);
1547
1548 fail:
1549         cpufreq_cpu_put(policy);
1550 }
1551
1552 static struct syscore_ops cpufreq_syscore_ops = {
1553         .suspend        = cpufreq_bp_suspend,
1554         .resume         = cpufreq_bp_resume,
1555 };
1556
1557 /**
1558  *      cpufreq_get_current_driver - return current driver's name
1559  *
1560  *      Return the name string of the currently loaded cpufreq driver
1561  *      or NULL, if none.
1562  */
1563 const char *cpufreq_get_current_driver(void)
1564 {
1565         if (cpufreq_driver)
1566                 return cpufreq_driver->name;
1567
1568         return NULL;
1569 }
1570 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1571
1572 /*********************************************************************
1573  *                     NOTIFIER LISTS INTERFACE                      *
1574  *********************************************************************/
1575
1576 /**
1577  *      cpufreq_register_notifier - register a driver with cpufreq
1578  *      @nb: notifier function to register
1579  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1580  *
1581  *      Add a driver to one of two lists: either a list of drivers that
1582  *      are notified about clock rate changes (once before and once after
1583  *      the transition), or a list of drivers that are notified about
1584  *      changes in cpufreq policy.
1585  *
1586  *      This function may sleep, and has the same return conditions as
1587  *      blocking_notifier_chain_register.
1588  */
1589 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1590 {
1591         int ret;
1592
1593         if (cpufreq_disabled())
1594                 return -EINVAL;
1595
1596         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1597
1598         switch (list) {
1599         case CPUFREQ_TRANSITION_NOTIFIER:
1600                 ret = srcu_notifier_chain_register(
1601                                 &cpufreq_transition_notifier_list, nb);
1602                 break;
1603         case CPUFREQ_POLICY_NOTIFIER:
1604                 ret = blocking_notifier_chain_register(
1605                                 &cpufreq_policy_notifier_list, nb);
1606                 break;
1607         default:
1608                 ret = -EINVAL;
1609         }
1610
1611         return ret;
1612 }
1613 EXPORT_SYMBOL(cpufreq_register_notifier);
1614
1615 /**
1616  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1617  *      @nb: notifier block to be unregistered
1618  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1619  *
1620  *      Remove a driver from the CPU frequency notifier list.
1621  *
1622  *      This function may sleep, and has the same return conditions as
1623  *      blocking_notifier_chain_unregister.
1624  */
1625 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1626 {
1627         int ret;
1628
1629         if (cpufreq_disabled())
1630                 return -EINVAL;
1631
1632         switch (list) {
1633         case CPUFREQ_TRANSITION_NOTIFIER:
1634                 ret = srcu_notifier_chain_unregister(
1635                                 &cpufreq_transition_notifier_list, nb);
1636                 break;
1637         case CPUFREQ_POLICY_NOTIFIER:
1638                 ret = blocking_notifier_chain_unregister(
1639                                 &cpufreq_policy_notifier_list, nb);
1640                 break;
1641         default:
1642                 ret = -EINVAL;
1643         }
1644
1645         return ret;
1646 }
1647 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1648
1649
1650 /*********************************************************************
1651  *                              GOVERNORS                            *
1652  *********************************************************************/
1653
1654 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1655                             unsigned int target_freq,
1656                             unsigned int relation)
1657 {
1658         int retval = -EINVAL;
1659         unsigned int old_target_freq = target_freq;
1660
1661         if (cpufreq_disabled())
1662                 return -ENODEV;
1663
1664         /* Make sure that target_freq is within supported range */
1665         if (target_freq > policy->max)
1666                 target_freq = policy->max;
1667         if (target_freq < policy->min)
1668                 target_freq = policy->min;
1669
1670         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1671                         policy->cpu, target_freq, relation, old_target_freq);
1672
1673         /*
1674          * This might look like a redundant call as we are checking it again
1675          * after finding index. But it is left intentionally for cases where
1676          * exactly same freq is called again and so we can save on few function
1677          * calls.
1678          */
1679         if (target_freq == policy->cur)
1680                 return 0;
1681
1682         if (cpufreq_driver->target)
1683                 retval = cpufreq_driver->target(policy, target_freq, relation);
1684         else if (cpufreq_driver->target_index) {
1685                 struct cpufreq_frequency_table *freq_table;
1686                 struct cpufreq_freqs freqs;
1687                 bool notify;
1688                 int index;
1689
1690                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1691                 if (unlikely(!freq_table)) {
1692                         pr_err("%s: Unable to find freq_table\n", __func__);
1693                         goto out;
1694                 }
1695
1696                 retval = cpufreq_frequency_table_target(policy, freq_table,
1697                                 target_freq, relation, &index);
1698                 if (unlikely(retval)) {
1699                         pr_err("%s: Unable to find matching freq\n", __func__);
1700                         goto out;
1701                 }
1702
1703                 if (freq_table[index].frequency == policy->cur) {
1704                         retval = 0;
1705                         goto out;
1706                 }
1707
1708                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1709
1710                 if (notify) {
1711                         freqs.old = policy->cur;
1712                         freqs.new = freq_table[index].frequency;
1713                         freqs.flags = 0;
1714
1715                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1716                                         __func__, policy->cpu, freqs.old,
1717                                         freqs.new);
1718
1719                         cpufreq_notify_transition(policy, &freqs,
1720                                         CPUFREQ_PRECHANGE);
1721                 }
1722
1723                 retval = cpufreq_driver->target_index(policy, index);
1724                 if (retval)
1725                         pr_err("%s: Failed to change cpu frequency: %d\n",
1726                                         __func__, retval);
1727
1728                 if (notify) {
1729                         /*
1730                          * Notify with old freq in case we failed to change
1731                          * frequency
1732                          */
1733                         if (retval)
1734                                 freqs.new = freqs.old;
1735
1736                         cpufreq_notify_transition(policy, &freqs,
1737                                         CPUFREQ_POSTCHANGE);
1738                 }
1739         }
1740
1741 out:
1742         return retval;
1743 }
1744 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1745
1746 int cpufreq_driver_target(struct cpufreq_policy *policy,
1747                           unsigned int target_freq,
1748                           unsigned int relation)
1749 {
1750         int ret = -EINVAL;
1751
1752         down_write(&policy->rwsem);
1753
1754         ret = __cpufreq_driver_target(policy, target_freq, relation);
1755
1756         up_write(&policy->rwsem);
1757
1758         return ret;
1759 }
1760 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1761
1762 /*
1763  * when "event" is CPUFREQ_GOV_LIMITS
1764  */
1765
1766 static int __cpufreq_governor(struct cpufreq_policy *policy,
1767                                         unsigned int event)
1768 {
1769         int ret;
1770
1771         /* Only must be defined when default governor is known to have latency
1772            restrictions, like e.g. conservative or ondemand.
1773            That this is the case is already ensured in Kconfig
1774         */
1775 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1776         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1777 #else
1778         struct cpufreq_governor *gov = NULL;
1779 #endif
1780
1781         if (policy->governor->max_transition_latency &&
1782             policy->cpuinfo.transition_latency >
1783             policy->governor->max_transition_latency) {
1784                 if (!gov)
1785                         return -EINVAL;
1786                 else {
1787                         printk(KERN_WARNING "%s governor failed, too long"
1788                                " transition latency of HW, fallback"
1789                                " to %s governor\n",
1790                                policy->governor->name,
1791                                gov->name);
1792                         policy->governor = gov;
1793                 }
1794         }
1795
1796         if (event == CPUFREQ_GOV_POLICY_INIT)
1797                 if (!try_module_get(policy->governor->owner))
1798                         return -EINVAL;
1799
1800         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1801                                                 policy->cpu, event);
1802
1803         mutex_lock(&cpufreq_governor_lock);
1804         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1805             || (!policy->governor_enabled
1806             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1807                 mutex_unlock(&cpufreq_governor_lock);
1808                 return -EBUSY;
1809         }
1810
1811         if (event == CPUFREQ_GOV_STOP)
1812                 policy->governor_enabled = false;
1813         else if (event == CPUFREQ_GOV_START)
1814                 policy->governor_enabled = true;
1815
1816         mutex_unlock(&cpufreq_governor_lock);
1817
1818         ret = policy->governor->governor(policy, event);
1819
1820         if (!ret) {
1821                 if (event == CPUFREQ_GOV_POLICY_INIT)
1822                         policy->governor->initialized++;
1823                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1824                         policy->governor->initialized--;
1825         } else {
1826                 /* Restore original values */
1827                 mutex_lock(&cpufreq_governor_lock);
1828                 if (event == CPUFREQ_GOV_STOP)
1829                         policy->governor_enabled = true;
1830                 else if (event == CPUFREQ_GOV_START)
1831                         policy->governor_enabled = false;
1832                 mutex_unlock(&cpufreq_governor_lock);
1833         }
1834
1835         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1836                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1837                 module_put(policy->governor->owner);
1838
1839         return ret;
1840 }
1841
1842 int cpufreq_register_governor(struct cpufreq_governor *governor)
1843 {
1844         int err;
1845
1846         if (!governor)
1847                 return -EINVAL;
1848
1849         if (cpufreq_disabled())
1850                 return -ENODEV;
1851
1852         mutex_lock(&cpufreq_governor_mutex);
1853
1854         governor->initialized = 0;
1855         err = -EBUSY;
1856         if (__find_governor(governor->name) == NULL) {
1857                 err = 0;
1858                 list_add(&governor->governor_list, &cpufreq_governor_list);
1859         }
1860
1861         mutex_unlock(&cpufreq_governor_mutex);
1862         return err;
1863 }
1864 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1865
1866 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1867 {
1868 #ifdef CONFIG_HOTPLUG_CPU
1869         int cpu;
1870 #endif
1871
1872         if (!governor)
1873                 return;
1874
1875         if (cpufreq_disabled())
1876                 return;
1877
1878 #ifdef CONFIG_HOTPLUG_CPU
1879         for_each_present_cpu(cpu) {
1880                 if (cpu_online(cpu))
1881                         continue;
1882                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1883                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1884         }
1885 #endif
1886
1887         mutex_lock(&cpufreq_governor_mutex);
1888         list_del(&governor->governor_list);
1889         mutex_unlock(&cpufreq_governor_mutex);
1890         return;
1891 }
1892 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1893
1894
1895 /*********************************************************************
1896  *                          POLICY INTERFACE                         *
1897  *********************************************************************/
1898
1899 /**
1900  * cpufreq_get_policy - get the current cpufreq_policy
1901  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1902  *      is written
1903  *
1904  * Reads the current cpufreq policy.
1905  */
1906 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1907 {
1908         struct cpufreq_policy *cpu_policy;
1909         if (!policy)
1910                 return -EINVAL;
1911
1912         cpu_policy = cpufreq_cpu_get(cpu);
1913         if (!cpu_policy)
1914                 return -EINVAL;
1915
1916         memcpy(policy, cpu_policy, sizeof(*policy));
1917
1918         cpufreq_cpu_put(cpu_policy);
1919         return 0;
1920 }
1921 EXPORT_SYMBOL(cpufreq_get_policy);
1922
1923 /*
1924  * policy : current policy.
1925  * new_policy: policy to be set.
1926  */
1927 static int cpufreq_set_policy(struct cpufreq_policy *policy,
1928                                 struct cpufreq_policy *new_policy)
1929 {
1930         int ret = 0, failed = 1;
1931
1932         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
1933                 new_policy->min, new_policy->max);
1934
1935         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
1936
1937         if (new_policy->min > policy->max || new_policy->max < policy->min) {
1938                 ret = -EINVAL;
1939                 goto error_out;
1940         }
1941
1942         /* verify the cpu speed can be set within this limit */
1943         ret = cpufreq_driver->verify(new_policy);
1944         if (ret)
1945                 goto error_out;
1946
1947         /* adjust if necessary - all reasons */
1948         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1949                         CPUFREQ_ADJUST, new_policy);
1950
1951         /* adjust if necessary - hardware incompatibility*/
1952         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1953                         CPUFREQ_INCOMPATIBLE, new_policy);
1954
1955         /*
1956          * verify the cpu speed can be set within this limit, which might be
1957          * different to the first one
1958          */
1959         ret = cpufreq_driver->verify(new_policy);
1960         if (ret)
1961                 goto error_out;
1962
1963         /* notification of the new policy */
1964         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1965                         CPUFREQ_NOTIFY, new_policy);
1966
1967         policy->min = new_policy->min;
1968         policy->max = new_policy->max;
1969
1970         pr_debug("new min and max freqs are %u - %u kHz\n",
1971                                         policy->min, policy->max);
1972
1973         if (cpufreq_driver->setpolicy) {
1974                 policy->policy = new_policy->policy;
1975                 pr_debug("setting range\n");
1976                 ret = cpufreq_driver->setpolicy(new_policy);
1977         } else {
1978                 if (new_policy->governor != policy->governor) {
1979                         /* save old, working values */
1980                         struct cpufreq_governor *old_gov = policy->governor;
1981
1982                         pr_debug("governor switch\n");
1983
1984                         /* end old governor */
1985                         if (policy->governor) {
1986                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1987                                 up_write(&policy->rwsem);
1988                                 __cpufreq_governor(policy,
1989                                                 CPUFREQ_GOV_POLICY_EXIT);
1990                                 down_write(&policy->rwsem);
1991                         }
1992
1993                         /* start new governor */
1994                         policy->governor = new_policy->governor;
1995                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
1996                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
1997                                         failed = 0;
1998                                 } else {
1999                                         up_write(&policy->rwsem);
2000                                         __cpufreq_governor(policy,
2001                                                         CPUFREQ_GOV_POLICY_EXIT);
2002                                         down_write(&policy->rwsem);
2003                                 }
2004                         }
2005
2006                         if (failed) {
2007                                 /* new governor failed, so re-start old one */
2008                                 pr_debug("starting governor %s failed\n",
2009                                                         policy->governor->name);
2010                                 if (old_gov) {
2011                                         policy->governor = old_gov;
2012                                         __cpufreq_governor(policy,
2013                                                         CPUFREQ_GOV_POLICY_INIT);
2014                                         __cpufreq_governor(policy,
2015                                                            CPUFREQ_GOV_START);
2016                                 }
2017                                 ret = -EINVAL;
2018                                 goto error_out;
2019                         }
2020                         /* might be a policy change, too, so fall through */
2021                 }
2022                 pr_debug("governor: change or update limits\n");
2023                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2024         }
2025
2026 error_out:
2027         return ret;
2028 }
2029
2030 /**
2031  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2032  *      @cpu: CPU which shall be re-evaluated
2033  *
2034  *      Useful for policy notifiers which have different necessities
2035  *      at different times.
2036  */
2037 int cpufreq_update_policy(unsigned int cpu)
2038 {
2039         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2040         struct cpufreq_policy new_policy;
2041         int ret;
2042
2043         if (!policy) {
2044                 ret = -ENODEV;
2045                 goto no_policy;
2046         }
2047
2048         down_write(&policy->rwsem);
2049
2050         pr_debug("updating policy for CPU %u\n", cpu);
2051         memcpy(&new_policy, policy, sizeof(*policy));
2052         new_policy.min = policy->user_policy.min;
2053         new_policy.max = policy->user_policy.max;
2054         new_policy.policy = policy->user_policy.policy;
2055         new_policy.governor = policy->user_policy.governor;
2056
2057         /*
2058          * BIOS might change freq behind our back
2059          * -> ask driver for current freq and notify governors about a change
2060          */
2061         if (cpufreq_driver->get) {
2062                 new_policy.cur = cpufreq_driver->get(cpu);
2063                 if (!policy->cur) {
2064                         pr_debug("Driver did not initialize current freq");
2065                         policy->cur = new_policy.cur;
2066                 } else {
2067                         if (policy->cur != new_policy.cur && has_target())
2068                                 cpufreq_out_of_sync(cpu, policy->cur,
2069                                                                 new_policy.cur);
2070                 }
2071         }
2072
2073         ret = cpufreq_set_policy(policy, &new_policy);
2074
2075         up_write(&policy->rwsem);
2076
2077         cpufreq_cpu_put(policy);
2078 no_policy:
2079         return ret;
2080 }
2081 EXPORT_SYMBOL(cpufreq_update_policy);
2082
2083 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2084                                         unsigned long action, void *hcpu)
2085 {
2086         unsigned int cpu = (unsigned long)hcpu;
2087         struct device *dev;
2088         bool frozen = false;
2089
2090         dev = get_cpu_device(cpu);
2091         if (dev) {
2092
2093                 if (action & CPU_TASKS_FROZEN)
2094                         frozen = true;
2095
2096                 switch (action & ~CPU_TASKS_FROZEN) {
2097                 case CPU_ONLINE:
2098                         __cpufreq_add_dev(dev, NULL, frozen);
2099                         cpufreq_update_policy(cpu);
2100                         break;
2101
2102                 case CPU_DOWN_PREPARE:
2103                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2104                         break;
2105
2106                 case CPU_POST_DEAD:
2107                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2108                         break;
2109
2110                 case CPU_DOWN_FAILED:
2111                         __cpufreq_add_dev(dev, NULL, frozen);
2112                         break;
2113                 }
2114         }
2115         return NOTIFY_OK;
2116 }
2117
2118 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2119         .notifier_call = cpufreq_cpu_callback,
2120 };
2121
2122 /*********************************************************************
2123  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2124  *********************************************************************/
2125
2126 /**
2127  * cpufreq_register_driver - register a CPU Frequency driver
2128  * @driver_data: A struct cpufreq_driver containing the values#
2129  * submitted by the CPU Frequency driver.
2130  *
2131  * Registers a CPU Frequency driver to this core code. This code
2132  * returns zero on success, -EBUSY when another driver got here first
2133  * (and isn't unregistered in the meantime).
2134  *
2135  */
2136 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2137 {
2138         unsigned long flags;
2139         int ret;
2140
2141         if (cpufreq_disabled())
2142                 return -ENODEV;
2143
2144         if (!driver_data || !driver_data->verify || !driver_data->init ||
2145             !(driver_data->setpolicy || driver_data->target_index ||
2146                     driver_data->target))
2147                 return -EINVAL;
2148
2149         pr_debug("trying to register driver %s\n", driver_data->name);
2150
2151         if (driver_data->setpolicy)
2152                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2153
2154         write_lock_irqsave(&cpufreq_driver_lock, flags);
2155         if (cpufreq_driver) {
2156                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2157                 return -EEXIST;
2158         }
2159         cpufreq_driver = driver_data;
2160         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2161
2162         ret = subsys_interface_register(&cpufreq_interface);
2163         if (ret)
2164                 goto err_null_driver;
2165
2166         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2167                 int i;
2168                 ret = -ENODEV;
2169
2170                 /* check for at least one working CPU */
2171                 for (i = 0; i < nr_cpu_ids; i++)
2172                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2173                                 ret = 0;
2174                                 break;
2175                         }
2176
2177                 /* if all ->init() calls failed, unregister */
2178                 if (ret) {
2179                         pr_debug("no CPU initialized for driver %s\n",
2180                                                         driver_data->name);
2181                         goto err_if_unreg;
2182                 }
2183         }
2184
2185         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2186         pr_debug("driver %s up and running\n", driver_data->name);
2187
2188         return 0;
2189 err_if_unreg:
2190         subsys_interface_unregister(&cpufreq_interface);
2191 err_null_driver:
2192         write_lock_irqsave(&cpufreq_driver_lock, flags);
2193         cpufreq_driver = NULL;
2194         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2195         return ret;
2196 }
2197 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2198
2199 /**
2200  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2201  *
2202  * Unregister the current CPUFreq driver. Only call this if you have
2203  * the right to do so, i.e. if you have succeeded in initialising before!
2204  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2205  * currently not initialised.
2206  */
2207 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2208 {
2209         unsigned long flags;
2210
2211         if (!cpufreq_driver || (driver != cpufreq_driver))
2212                 return -EINVAL;
2213
2214         pr_debug("unregistering driver %s\n", driver->name);
2215
2216         subsys_interface_unregister(&cpufreq_interface);
2217         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2218
2219         down_write(&cpufreq_rwsem);
2220         write_lock_irqsave(&cpufreq_driver_lock, flags);
2221
2222         cpufreq_driver = NULL;
2223
2224         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2225         up_write(&cpufreq_rwsem);
2226
2227         return 0;
2228 }
2229 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2230
2231 static int __init cpufreq_core_init(void)
2232 {
2233         if (cpufreq_disabled())
2234                 return -ENODEV;
2235
2236         cpufreq_global_kobject = kobject_create();
2237         BUG_ON(!cpufreq_global_kobject);
2238         register_syscore_ops(&cpufreq_syscore_ops);
2239
2240         return 0;
2241 }
2242 core_initcall(cpufreq_core_init);