drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 static DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 /*
  51  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  52  * all cpufreq/hotplug/workqueue/etc related lock issues.
  53  *
  54  * The rules for this semaphore:
  55  * - Any routine that wants to read from the policy structure will
  56  *   do a down_read on this semaphore.
  57  * - Any routine that will write to the policy structure and/or may take away
  58  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  59  *   mode before doing so.
  60  *
  61  * Additional rules:
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  68
  69 #define lock_policy_rwsem(mode, cpu)                                    \
  70 static int lock_policy_rwsem_##mode(int cpu)                            \
  71 {                                                                       \
  72         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \
  73         BUG_ON(!policy);                                                \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu));           \
  75                                                                         \
  76         return 0;                                                       \
  77 }
  78
  79 lock_policy_rwsem(read, cpu);
  80 lock_policy_rwsem(write, cpu);
  81
  82 #define unlock_policy_rwsem(mode, cpu)                                  \
  83 static void unlock_policy_rwsem_##mode(int cpu)                         \
  84 {                                                                       \
  85         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \
  86         BUG_ON(!policy);                                                \
  87         up_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu));             \
  88 }
  89
  90 unlock_policy_rwsem(read, cpu);
  91 unlock_policy_rwsem(write, cpu);
  92
  93 /*
  94  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  95  * sections
  96  */
  97 static DECLARE_RWSEM(cpufreq_rwsem);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 187 {
 188         struct cpufreq_policy *policy = NULL;
 189         unsigned long flags;
 190
 191         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 192                 return NULL;
 193
 194         if (!down_read_trylock(&cpufreq_rwsem))
 195                 return NULL;
 196
 197         /* get the cpufreq driver */
 198         read_lock_irqsave(&cpufreq_driver_lock, flags);
 199
 200         if (cpufreq_driver) {
 201                 /* get the CPU */
 202                 policy = per_cpu(cpufreq_cpu_data, cpu);
 203                 if (policy)
 204                         kobject_get(&policy->kobj);
 205         }
 206
 207         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 208
 209         if (!policy)
 210                 up_read(&cpufreq_rwsem);
 211
 212         return policy;
 213 }
 214 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 215
 216 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 217 {
 218         if (cpufreq_disabled())
 219                 return;
 220
 221         kobject_put(&policy->kobj);
 222         up_read(&cpufreq_rwsem);
 223 }
 224 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 225
 226 /*********************************************************************
 227  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 228  *********************************************************************/
 229
 230 /**
 231  * adjust_jiffies - adjust the system "loops_per_jiffy"
 232  *
 233  * This function alters the system "loops_per_jiffy" for the clock
 234  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 235  * systems as each CPU might be scaled differently. So, use the arch
 236  * per-CPU loops_per_jiffy value wherever possible.
 237  */
 238 #ifndef CONFIG_SMP
 239 static unsigned long l_p_j_ref;
 240 static unsigned int l_p_j_ref_freq;
 241
 242 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 243 {
 244         if (ci->flags & CPUFREQ_CONST_LOOPS)
 245                 return;
 246
 247         if (!l_p_j_ref_freq) {
 248                 l_p_j_ref = loops_per_jiffy;
 249                 l_p_j_ref_freq = ci->old;
 250                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 251                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 252         }
 253         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 254             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 255                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 256                                                                 ci->new);
 257                 pr_debug("scaling loops_per_jiffy to %lu "
 258                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 259         }
 260 }
 261 #else
 262 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 263 {
 264         return;
 265 }
 266 #endif
 267
 268 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 269                 struct cpufreq_freqs *freqs, unsigned int state)
 270 {
 271         BUG_ON(irqs_disabled());
 272
 273         if (cpufreq_disabled())
 274                 return;
 275
 276         freqs->flags = cpufreq_driver->flags;
 277         pr_debug("notification %u of frequency transition to %u kHz\n",
 278                 state, freqs->new);
 279
 280         switch (state) {
 281
 282         case CPUFREQ_PRECHANGE:
 283                 if (WARN(policy->transition_ongoing ==
 284                                         cpumask_weight(policy->cpus),
 285                                 "In middle of another frequency transition\n"))
 286                         return;
 287
 288                 policy->transition_ongoing++;
 289
 290                 /* detect if the driver reported a value as "old frequency"
 291                  * which is not equal to what the cpufreq core thinks is
 292                  * "old frequency".
 293                  */
 294                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 295                         if ((policy) && (policy->cpu == freqs->cpu) &&
 296                             (policy->cur) && (policy->cur != freqs->old)) {
 297                                 pr_debug("Warning: CPU frequency is"
 298                                         " %u, cpufreq assumed %u kHz.\n",
 299                                         freqs->old, policy->cur);
 300                                 freqs->old = policy->cur;
 301                         }
 302                 }
 303                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 304                                 CPUFREQ_PRECHANGE, freqs);
 305                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 306                 break;
 307
 308         case CPUFREQ_POSTCHANGE:
 309                 if (WARN(!policy->transition_ongoing,
 310                                 "No frequency transition in progress\n"))
 311                         return;
 312
 313                 policy->transition_ongoing--;
 314
 315                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 316                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 317                         (unsigned long)freqs->cpu);
 318                 trace_cpu_frequency(freqs->new, freqs->cpu);
 319                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 320                                 CPUFREQ_POSTCHANGE, freqs);
 321                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 322                         policy->cur = freqs->new;
 323                 break;
 324         }
 325 }
 326
 327 /**
 328  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 329  * on frequency transition.
 330  *
 331  * This function calls the transition notifiers and the "adjust_jiffies"
 332  * function. It is called twice on all CPU frequency changes that have
 333  * external effects.
 334  */
 335 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 336                 struct cpufreq_freqs *freqs, unsigned int state)
 337 {
 338         for_each_cpu(freqs->cpu, policy->cpus)
 339                 __cpufreq_notify_transition(policy, freqs, state);
 340 }
 341 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 342
 343
 344 /*********************************************************************
 345  *                          SYSFS INTERFACE                          *
 346  *********************************************************************/
 347
 348 static struct cpufreq_governor *__find_governor(const char *str_governor)
 349 {
 350         struct cpufreq_governor *t;
 351
 352         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 353                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 354                         return t;
 355
 356         return NULL;
 357 }
 358
 359 /**
 360  * cpufreq_parse_governor - parse a governor string
 361  */
 362 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 363                                 struct cpufreq_governor **governor)
 364 {
 365         int err = -EINVAL;
 366
 367         if (!cpufreq_driver)
 368                 goto out;
 369
 370         if (cpufreq_driver->setpolicy) {
 371                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 372                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 373                         err = 0;
 374                 } else if (!strnicmp(str_governor, "powersave",
 375                                                 CPUFREQ_NAME_LEN)) {
 376                         *policy = CPUFREQ_POLICY_POWERSAVE;
 377                         err = 0;
 378                 }
 379         } else if (cpufreq_driver->target) {
 380                 struct cpufreq_governor *t;
 381
 382                 mutex_lock(&cpufreq_governor_mutex);
 383
 384                 t = __find_governor(str_governor);
 385
 386                 if (t == NULL) {
 387                         int ret;
 388
 389                         mutex_unlock(&cpufreq_governor_mutex);
 390                         ret = request_module("cpufreq_%s", str_governor);
 391                         mutex_lock(&cpufreq_governor_mutex);
 392
 393                         if (ret == 0)
 394                                 t = __find_governor(str_governor);
 395                 }
 396
 397                 if (t != NULL) {
 398                         *governor = t;
 399                         err = 0;
 400                 }
 401
 402                 mutex_unlock(&cpufreq_governor_mutex);
 403         }
 404 out:
 405         return err;
 406 }
 407
 408 /**
 409  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 410  * print out cpufreq information
 411  *
 412  * Write out information from cpufreq_driver->policy[cpu]; object must be
 413  * "unsigned int".
 414  */
 415
 416 #define show_one(file_name, object)                     \
 417 static ssize_t show_##file_name                         \
 418 (struct cpufreq_policy *policy, char *buf)              \
 419 {                                                       \
 420         return sprintf(buf, "%u\n", policy->object);    \
 421 }
 422
 423 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 424 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 425 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 426 show_one(scaling_min_freq, min);
 427 show_one(scaling_max_freq, max);
 428 show_one(scaling_cur_freq, cur);
 429
 430 static int __cpufreq_set_policy(struct cpufreq_policy *policy,
 431                                 struct cpufreq_policy *new_policy);
 432
 433 /**
 434  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 435  */
 436 #define store_one(file_name, object)                    \
 437 static ssize_t store_##file_name                                        \
 438 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 439 {                                                                       \
 440         unsigned int ret;                                               \
 441         struct cpufreq_policy new_policy;                               \
 442                                                                         \
 443         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 444         if (ret)                                                        \
 445                 return -EINVAL;                                         \
 446                                                                         \
 447         ret = sscanf(buf, "%u", &new_policy.object);                    \
 448         if (ret != 1)                                                   \
 449                 return -EINVAL;                                         \
 450                                                                         \
 451         ret = __cpufreq_set_policy(policy, &new_policy);                \
 452         policy->user_policy.object = policy->object;                    \
 453                                                                         \
 454         return ret ? ret : count;                                       \
 455 }
 456
 457 store_one(scaling_min_freq, min);
 458 store_one(scaling_max_freq, max);
 459
 460 /**
 461  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 462  */
 463 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 464                                         char *buf)
 465 {
 466         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 467         if (!cur_freq)
 468                 return sprintf(buf, "<unknown>");
 469         return sprintf(buf, "%u\n", cur_freq);
 470 }
 471
 472 /**
 473  * show_scaling_governor - show the current policy for the specified CPU
 474  */
 475 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 476 {
 477         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 478                 return sprintf(buf, "powersave\n");
 479         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 480                 return sprintf(buf, "performance\n");
 481         else if (policy->governor)
 482                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 483                                 policy->governor->name);
 484         return -EINVAL;
 485 }
 486
 487 /**
 488  * store_scaling_governor - store policy for the specified CPU
 489  */
 490 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 491                                         const char *buf, size_t count)
 492 {
 493         unsigned int ret;
 494         char    str_governor[16];
 495         struct cpufreq_policy new_policy;
 496
 497         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 498         if (ret)
 499                 return ret;
 500
 501         ret = sscanf(buf, "%15s", str_governor);
 502         if (ret != 1)
 503                 return -EINVAL;
 504
 505         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 506                                                 &new_policy.governor))
 507                 return -EINVAL;
 508
 509         /*
 510          * Do not use cpufreq_set_policy here or the user_policy.max
 511          * will be wrongly overridden
 512          */
 513         ret = __cpufreq_set_policy(policy, &new_policy);
 514
 515         policy->user_policy.policy = policy->policy;
 516         policy->user_policy.governor = policy->governor;
 517
 518         if (ret)
 519                 return ret;
 520         else
 521                 return count;
 522 }
 523
 524 /**
 525  * show_scaling_driver - show the cpufreq driver currently loaded
 526  */
 527 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 528 {
 529         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 530 }
 531
 532 /**
 533  * show_scaling_available_governors - show the available CPUfreq governors
 534  */
 535 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 536                                                 char *buf)
 537 {
 538         ssize_t i = 0;
 539         struct cpufreq_governor *t;
 540
 541         if (!cpufreq_driver->target) {
 542                 i += sprintf(buf, "performance powersave");
 543                 goto out;
 544         }
 545
 546         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 547                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 548                     - (CPUFREQ_NAME_LEN + 2)))
 549                         goto out;
 550                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 551         }
 552 out:
 553         i += sprintf(&buf[i], "\n");
 554         return i;
 555 }
 556
 557 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 558 {
 559         ssize_t i = 0;
 560         unsigned int cpu;
 561
 562         for_each_cpu(cpu, mask) {
 563                 if (i)
 564                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 565                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 566                 if (i >= (PAGE_SIZE - 5))
 567                         break;
 568         }
 569         i += sprintf(&buf[i], "\n");
 570         return i;
 571 }
 572 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 573
 574 /**
 575  * show_related_cpus - show the CPUs affected by each transition even if
 576  * hw coordination is in use
 577  */
 578 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 579 {
 580         return cpufreq_show_cpus(policy->related_cpus, buf);
 581 }
 582
 583 /**
 584  * show_affected_cpus - show the CPUs affected by each transition
 585  */
 586 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 587 {
 588         return cpufreq_show_cpus(policy->cpus, buf);
 589 }
 590
 591 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 592                                         const char *buf, size_t count)
 593 {
 594         unsigned int freq = 0;
 595         unsigned int ret;
 596
 597         if (!policy->governor || !policy->governor->store_setspeed)
 598                 return -EINVAL;
 599
 600         ret = sscanf(buf, "%u", &freq);
 601         if (ret != 1)
 602                 return -EINVAL;
 603
 604         policy->governor->store_setspeed(policy, freq);
 605
 606         return count;
 607 }
 608
 609 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 610 {
 611         if (!policy->governor || !policy->governor->show_setspeed)
 612                 return sprintf(buf, "<unsupported>\n");
 613
 614         return policy->governor->show_setspeed(policy, buf);
 615 }
 616
 617 /**
 618  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 619  */
 620 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 621 {
 622         unsigned int limit;
 623         int ret;
 624         if (cpufreq_driver->bios_limit) {
 625                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 626                 if (!ret)
 627                         return sprintf(buf, "%u\n", limit);
 628         }
 629         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 630 }
 631
 632 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 633 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 634 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 635 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 636 cpufreq_freq_attr_ro(scaling_available_governors);
 637 cpufreq_freq_attr_ro(scaling_driver);
 638 cpufreq_freq_attr_ro(scaling_cur_freq);
 639 cpufreq_freq_attr_ro(bios_limit);
 640 cpufreq_freq_attr_ro(related_cpus);
 641 cpufreq_freq_attr_ro(affected_cpus);
 642 cpufreq_freq_attr_rw(scaling_min_freq);
 643 cpufreq_freq_attr_rw(scaling_max_freq);
 644 cpufreq_freq_attr_rw(scaling_governor);
 645 cpufreq_freq_attr_rw(scaling_setspeed);
 646
 647 static struct attribute *default_attrs[] = {
 648         &cpuinfo_min_freq.attr,
 649         &cpuinfo_max_freq.attr,
 650         &cpuinfo_transition_latency.attr,
 651         &scaling_min_freq.attr,
 652         &scaling_max_freq.attr,
 653         &affected_cpus.attr,
 654         &related_cpus.attr,
 655         &scaling_governor.attr,
 656         &scaling_driver.attr,
 657         &scaling_available_governors.attr,
 658         &scaling_setspeed.attr,
 659         NULL
 660 };
 661
 662 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 663 #define to_attr(a) container_of(a, struct freq_attr, attr)
 664
 665 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 666 {
 667         struct cpufreq_policy *policy = to_policy(kobj);
 668         struct freq_attr *fattr = to_attr(attr);
 669         ssize_t ret = -EINVAL;
 670
 671         if (!down_read_trylock(&cpufreq_rwsem))
 672                 goto exit;
 673
 674         if (lock_policy_rwsem_read(policy->cpu) < 0)
 675                 goto up_read;
 676
 677         if (fattr->show)
 678                 ret = fattr->show(policy, buf);
 679         else
 680                 ret = -EIO;
 681
 682         unlock_policy_rwsem_read(policy->cpu);
 683
 684 up_read:
 685         up_read(&cpufreq_rwsem);
 686 exit:
 687         return ret;
 688 }
 689
 690 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 691                      const char *buf, size_t count)
 692 {
 693         struct cpufreq_policy *policy = to_policy(kobj);
 694         struct freq_attr *fattr = to_attr(attr);
 695         ssize_t ret = -EINVAL;
 696
 697         if (!down_read_trylock(&cpufreq_rwsem))
 698                 goto exit;
 699
 700         if (lock_policy_rwsem_write(policy->cpu) < 0)
 701                 goto up_read;
 702
 703         if (fattr->store)
 704                 ret = fattr->store(policy, buf, count);
 705         else
 706                 ret = -EIO;
 707
 708         unlock_policy_rwsem_write(policy->cpu);
 709
 710 up_read:
 711         up_read(&cpufreq_rwsem);
 712 exit:
 713         return ret;
 714 }
 715
 716 static void cpufreq_sysfs_release(struct kobject *kobj)
 717 {
 718         struct cpufreq_policy *policy = to_policy(kobj);
 719         pr_debug("last reference is dropped\n");
 720         complete(&policy->kobj_unregister);
 721 }
 722
 723 static const struct sysfs_ops sysfs_ops = {
 724         .show   = show,
 725         .store  = store,
 726 };
 727
 728 static struct kobj_type ktype_cpufreq = {
 729         .sysfs_ops      = &sysfs_ops,
 730         .default_attrs  = default_attrs,
 731         .release        = cpufreq_sysfs_release,
 732 };
 733
 734 struct kobject *cpufreq_global_kobject;
 735 EXPORT_SYMBOL(cpufreq_global_kobject);
 736
 737 static int cpufreq_global_kobject_usage;
 738
 739 int cpufreq_get_global_kobject(void)
 740 {
 741         if (!cpufreq_global_kobject_usage++)
 742                 return kobject_add(cpufreq_global_kobject,
 743                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 744
 745         return 0;
 746 }
 747 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 748
 749 void cpufreq_put_global_kobject(void)
 750 {
 751         if (!--cpufreq_global_kobject_usage)
 752                 kobject_del(cpufreq_global_kobject);
 753 }
 754 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 755
 756 int cpufreq_sysfs_create_file(const struct attribute *attr)
 757 {
 758         int ret = cpufreq_get_global_kobject();
 759
 760         if (!ret) {
 761                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 762                 if (ret)
 763                         cpufreq_put_global_kobject();
 764         }
 765
 766         return ret;
 767 }
 768 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 769
 770 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 771 {
 772         sysfs_remove_file(cpufreq_global_kobject, attr);
 773         cpufreq_put_global_kobject();
 774 }
 775 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 776
 777 /* symlink affected CPUs */
 778 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 779 {
 780         unsigned int j;
 781         int ret = 0;
 782
 783         for_each_cpu(j, policy->cpus) {
 784                 struct device *cpu_dev;
 785
 786                 if (j == policy->cpu)
 787                         continue;
 788
 789                 pr_debug("Adding link for CPU: %u\n", j);
 790                 cpu_dev = get_cpu_device(j);
 791                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 792                                         "cpufreq");
 793                 if (ret)
 794                         break;
 795         }
 796         return ret;
 797 }
 798
 799 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 800                                      struct device *dev)
 801 {
 802         struct freq_attr **drv_attr;
 803         int ret = 0;
 804
 805         /* prepare interface data */
 806         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 807                                    &dev->kobj, "cpufreq");
 808         if (ret)
 809                 return ret;
 810
 811         /* set up files for this cpu device */
 812         drv_attr = cpufreq_driver->attr;
 813         while ((drv_attr) && (*drv_attr)) {
 814                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 815                 if (ret)
 816                         goto err_out_kobj_put;
 817                 drv_attr++;
 818         }
 819         if (cpufreq_driver->get) {
 820                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 821                 if (ret)
 822                         goto err_out_kobj_put;
 823         }
 824         if (cpufreq_driver->target) {
 825                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 826                 if (ret)
 827                         goto err_out_kobj_put;
 828         }
 829         if (cpufreq_driver->bios_limit) {
 830                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 831                 if (ret)
 832                         goto err_out_kobj_put;
 833         }
 834
 835         ret = cpufreq_add_dev_symlink(policy);
 836         if (ret)
 837                 goto err_out_kobj_put;
 838
 839         return ret;
 840
 841 err_out_kobj_put:
 842         kobject_put(&policy->kobj);
 843         wait_for_completion(&policy->kobj_unregister);
 844         return ret;
 845 }
 846
 847 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 848 {
 849         struct cpufreq_policy new_policy;
 850         int ret = 0;
 851
 852         memcpy(&new_policy, policy, sizeof(*policy));
 853         /* assure that the starting sequence is run in __cpufreq_set_policy */
 854         policy->governor = NULL;
 855
 856         /* set default policy */
 857         ret = __cpufreq_set_policy(policy, &new_policy);
 858         policy->user_policy.policy = policy->policy;
 859         policy->user_policy.governor = policy->governor;
 860
 861         if (ret) {
 862                 pr_debug("setting policy failed\n");
 863                 if (cpufreq_driver->exit)
 864                         cpufreq_driver->exit(policy);
 865         }
 866 }
 867
 868 #ifdef CONFIG_HOTPLUG_CPU
 869 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 870                                   unsigned int cpu, struct device *dev,
 871                                   bool frozen)
 872 {
 873         int ret = 0, has_target = !!cpufreq_driver->target;
 874         unsigned long flags;
 875
 876         if (has_target) {
 877                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 878                 if (ret) {
 879                         pr_err("%s: Failed to stop governor\n", __func__);
 880                         return ret;
 881                 }
 882         }
 883
 884         lock_policy_rwsem_write(policy->cpu);
 885
 886         write_lock_irqsave(&cpufreq_driver_lock, flags);
 887
 888         cpumask_set_cpu(cpu, policy->cpus);
 889         per_cpu(cpufreq_cpu_data, cpu) = policy;
 890         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 891
 892         unlock_policy_rwsem_write(policy->cpu);
 893
 894         if (has_target) {
 895                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 896                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 897                         pr_err("%s: Failed to start governor\n", __func__);
 898                         return ret;
 899                 }
 900         }
 901
 902         /* Don't touch sysfs links during light-weight init */
 903         if (!frozen)
 904                 ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 905
 906         return ret;
 907 }
 908 #endif
 909
 910 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 911 {
 912         struct cpufreq_policy *policy;
 913         unsigned long flags;
 914
 915         write_lock_irqsave(&cpufreq_driver_lock, flags);
 916
 917         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 918
 919         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 920
 921         return policy;
 922 }
 923
 924 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 925 {
 926         struct cpufreq_policy *policy;
 927
 928         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 929         if (!policy)
 930                 return NULL;
 931
 932         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 933                 goto err_free_policy;
 934
 935         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 936                 goto err_free_cpumask;
 937
 938         INIT_LIST_HEAD(&policy->policy_list);
 939         return policy;
 940
 941 err_free_cpumask:
 942         free_cpumask_var(policy->cpus);
 943 err_free_policy:
 944         kfree(policy);
 945
 946         return NULL;
 947 }
 948
 949 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 950 {
 951         free_cpumask_var(policy->related_cpus);
 952         free_cpumask_var(policy->cpus);
 953         kfree(policy);
 954 }
 955
 956 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 957                              bool frozen)
 958 {
 959         unsigned int j, cpu = dev->id;
 960         int ret = -ENOMEM;
 961         struct cpufreq_policy *policy;
 962         unsigned long flags;
 963 #ifdef CONFIG_HOTPLUG_CPU
 964         struct cpufreq_policy *tpolicy;
 965         struct cpufreq_governor *gov;
 966 #endif
 967
 968         if (cpu_is_offline(cpu))
 969                 return 0;
 970
 971         pr_debug("adding CPU %u\n", cpu);
 972
 973 #ifdef CONFIG_SMP
 974         /* check whether a different CPU already registered this
 975          * CPU because it is in the same boat. */
 976         policy = cpufreq_cpu_get(cpu);
 977         if (unlikely(policy)) {
 978                 cpufreq_cpu_put(policy);
 979                 return 0;
 980         }
 981 #endif
 982
 983         if (!down_read_trylock(&cpufreq_rwsem))
 984                 return 0;
 985
 986 #ifdef CONFIG_HOTPLUG_CPU
 987         /* Check if this cpu was hot-unplugged earlier and has siblings */
 988         read_lock_irqsave(&cpufreq_driver_lock, flags);
 989         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
 990                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
 991                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 992                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev, frozen);
 993                         up_read(&cpufreq_rwsem);
 994                         return ret;
 995                 }
 996         }
 997         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 998 #endif
 999
1000         if (frozen)
1001                 /* Restore the saved policy when doing light-weight init */
1002                 policy = cpufreq_policy_restore(cpu);
1003         else
1004                 policy = cpufreq_policy_alloc();
1005
1006         if (!policy)
1007                 goto nomem_out;
1008
1009         policy->cpu = cpu;
1010         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1011         cpumask_copy(policy->cpus, cpumask_of(cpu));
1012
1013         init_completion(&policy->kobj_unregister);
1014         INIT_WORK(&policy->update, handle_update);
1015
1016         /* call driver. From then on the cpufreq must be able
1017          * to accept all calls to ->verify and ->setpolicy for this CPU
1018          */
1019         ret = cpufreq_driver->init(policy);
1020         if (ret) {
1021                 pr_debug("initialization failed\n");
1022                 goto err_set_policy_cpu;
1023         }
1024
1025         /* related cpus should atleast have policy->cpus */
1026         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1027
1028         /*
1029          * affected cpus must always be the one, which are online. We aren't
1030          * managing offline cpus here.
1031          */
1032         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1033
1034         policy->user_policy.min = policy->min;
1035         policy->user_policy.max = policy->max;
1036
1037         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1038                                      CPUFREQ_START, policy);
1039
1040 #ifdef CONFIG_HOTPLUG_CPU
1041         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1042         if (gov) {
1043                 policy->governor = gov;
1044                 pr_debug("Restoring governor %s for cpu %d\n",
1045                        policy->governor->name, cpu);
1046         }
1047 #endif
1048
1049         write_lock_irqsave(&cpufreq_driver_lock, flags);
1050         for_each_cpu(j, policy->cpus)
1051                 per_cpu(cpufreq_cpu_data, j) = policy;
1052         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1053
1054         if (!frozen) {
1055                 ret = cpufreq_add_dev_interface(policy, dev);
1056                 if (ret)
1057                         goto err_out_unregister;
1058         }
1059
1060         write_lock_irqsave(&cpufreq_driver_lock, flags);
1061         list_add(&policy->policy_list, &cpufreq_policy_list);
1062         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1063
1064         cpufreq_init_policy(policy);
1065
1066         kobject_uevent(&policy->kobj, KOBJ_ADD);
1067         up_read(&cpufreq_rwsem);
1068
1069         pr_debug("initialization complete\n");
1070
1071         return 0;
1072
1073 err_out_unregister:
1074         write_lock_irqsave(&cpufreq_driver_lock, flags);
1075         for_each_cpu(j, policy->cpus)
1076                 per_cpu(cpufreq_cpu_data, j) = NULL;
1077         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1078
1079 err_set_policy_cpu:
1080         cpufreq_policy_free(policy);
1081 nomem_out:
1082         up_read(&cpufreq_rwsem);
1083
1084         return ret;
1085 }
1086
1087 /**
1088  * cpufreq_add_dev - add a CPU device
1089  *
1090  * Adds the cpufreq interface for a CPU device.
1091  *
1092  * The Oracle says: try running cpufreq registration/unregistration concurrently
1093  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1094  * mess up, but more thorough testing is needed. - Mathieu
1095  */
1096 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1097 {
1098         return __cpufreq_add_dev(dev, sif, false);
1099 }
1100
1101 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1102 {
1103         policy->last_cpu = policy->cpu;
1104         policy->cpu = cpu;
1105
1106 #ifdef CONFIG_CPU_FREQ_TABLE
1107         cpufreq_frequency_table_update_policy_cpu(policy);
1108 #endif
1109         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1110                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1111 }
1112
1113 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1114                                            unsigned int old_cpu, bool frozen)
1115 {
1116         struct device *cpu_dev;
1117         int ret;
1118
1119         /* first sibling now owns the new sysfs dir */
1120         cpu_dev = get_cpu_device(cpumask_first(policy->cpus));
1121
1122         /* Don't touch sysfs files during light-weight tear-down */
1123         if (frozen)
1124                 return cpu_dev->id;
1125
1126         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1127         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1128         if (ret) {
1129                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1130
1131                 WARN_ON(lock_policy_rwsem_write(old_cpu));
1132                 cpumask_set_cpu(old_cpu, policy->cpus);
1133                 unlock_policy_rwsem_write(old_cpu);
1134
1135                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1136                                         "cpufreq");
1137
1138                 return -EINVAL;
1139         }
1140
1141         return cpu_dev->id;
1142 }
1143
1144 /**
1145  * __cpufreq_remove_dev - remove a CPU device
1146  *
1147  * Removes the cpufreq interface for a CPU device.
1148  * Caller should already have policy_rwsem in write mode for this CPU.
1149  * This routine frees the rwsem before returning.
1150  */
1151 static int __cpufreq_remove_dev(struct device *dev,
1152                                 struct subsys_interface *sif, bool frozen)
1153 {
1154         unsigned int cpu = dev->id, cpus;
1155         int new_cpu, ret;
1156         unsigned long flags;
1157         struct cpufreq_policy *policy;
1158         struct kobject *kobj;
1159         struct completion *cmp;
1160
1161         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1162
1163         write_lock_irqsave(&cpufreq_driver_lock, flags);
1164
1165         policy = per_cpu(cpufreq_cpu_data, cpu);
1166
1167         /* Save the policy somewhere when doing a light-weight tear-down */
1168         if (frozen)
1169                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1170
1171         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1172
1173         if (!policy) {
1174                 pr_debug("%s: No cpu_data found\n", __func__);
1175                 return -EINVAL;
1176         }
1177
1178         if (cpufreq_driver->target) {
1179                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1180                 if (ret) {
1181                         pr_err("%s: Failed to stop governor\n", __func__);
1182                         return ret;
1183                 }
1184         }
1185
1186 #ifdef CONFIG_HOTPLUG_CPU
1187         if (!cpufreq_driver->setpolicy)
1188                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1189                         policy->governor->name, CPUFREQ_NAME_LEN);
1190 #endif
1191
1192         WARN_ON(lock_policy_rwsem_write(cpu));
1193         cpus = cpumask_weight(policy->cpus);
1194
1195         if (cpus > 1)
1196                 cpumask_clear_cpu(cpu, policy->cpus);
1197         unlock_policy_rwsem_write(cpu);
1198
1199         if (cpu != policy->cpu && !frozen) {
1200                 sysfs_remove_link(&dev->kobj, "cpufreq");
1201         } else if (cpus > 1) {
1202
1203                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu, frozen);
1204                 if (new_cpu >= 0) {
1205                         WARN_ON(lock_policy_rwsem_write(cpu));
1206                         update_policy_cpu(policy, new_cpu);
1207                         unlock_policy_rwsem_write(cpu);
1208
1209                         if (!frozen) {
1210                                 pr_debug("%s: policy Kobject moved to cpu: %d "
1211                                          "from: %d\n",__func__, new_cpu, cpu);
1212                         }
1213                 }
1214         }
1215
1216         /* If cpu is last user of policy, free policy */
1217         if (cpus == 1) {
1218                 if (cpufreq_driver->target) {
1219                         ret = __cpufreq_governor(policy,
1220                                         CPUFREQ_GOV_POLICY_EXIT);
1221                         if (ret) {
1222                                 pr_err("%s: Failed to exit governor\n",
1223                                                 __func__);
1224                                 return ret;
1225                         }
1226                 }
1227
1228                 if (!frozen) {
1229                         lock_policy_rwsem_read(cpu);
1230                         kobj = &policy->kobj;
1231                         cmp = &policy->kobj_unregister;
1232                         unlock_policy_rwsem_read(cpu);
1233                         kobject_put(kobj);
1234
1235                         /*
1236                          * We need to make sure that the underlying kobj is
1237                          * actually not referenced anymore by anybody before we
1238                          * proceed with unloading.
1239                          */
1240                         pr_debug("waiting for dropping of refcount\n");
1241                         wait_for_completion(cmp);
1242                         pr_debug("wait complete\n");
1243                 }
1244
1245                 /*
1246                  * Perform the ->exit() even during light-weight tear-down,
1247                  * since this is a core component, and is essential for the
1248                  * subsequent light-weight ->init() to succeed.
1249                  */
1250                 if (cpufreq_driver->exit)
1251                         cpufreq_driver->exit(policy);
1252
1253                 /* Remove policy from list of active policies */
1254                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1255                 list_del(&policy->policy_list);
1256                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1257
1258                 if (!frozen)
1259                         cpufreq_policy_free(policy);
1260         } else {
1261                 if (cpufreq_driver->target) {
1262                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1263                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1264                                 pr_err("%s: Failed to start governor\n",
1265                                                 __func__);
1266                                 return ret;
1267                         }
1268                 }
1269         }
1270
1271         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1272         return 0;
1273 }
1274
1275 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1276 {
1277         unsigned int cpu = dev->id;
1278         int retval;
1279
1280         if (cpu_is_offline(cpu))
1281                 return 0;
1282
1283         retval = __cpufreq_remove_dev(dev, sif, false);
1284         return retval;
1285 }
1286
1287 static void handle_update(struct work_struct *work)
1288 {
1289         struct cpufreq_policy *policy =
1290                 container_of(work, struct cpufreq_policy, update);
1291         unsigned int cpu = policy->cpu;
1292         pr_debug("handle_update for cpu %u called\n", cpu);
1293         cpufreq_update_policy(cpu);
1294 }
1295
1296 /**
1297  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1298  *      in deep trouble.
1299  *      @cpu: cpu number
1300  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1301  *      @new_freq: CPU frequency the CPU actually runs at
1302  *
1303  *      We adjust to current frequency first, and need to clean up later.
1304  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1305  */
1306 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1307                                 unsigned int new_freq)
1308 {
1309         struct cpufreq_policy *policy;
1310         struct cpufreq_freqs freqs;
1311         unsigned long flags;
1312
1313         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1314                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1315
1316         freqs.old = old_freq;
1317         freqs.new = new_freq;
1318
1319         read_lock_irqsave(&cpufreq_driver_lock, flags);
1320         policy = per_cpu(cpufreq_cpu_data, cpu);
1321         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1322
1323         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1324         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1325 }
1326
1327 /**
1328  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1329  * @cpu: CPU number
1330  *
1331  * This is the last known freq, without actually getting it from the driver.
1332  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1333  */
1334 unsigned int cpufreq_quick_get(unsigned int cpu)
1335 {
1336         struct cpufreq_policy *policy;
1337         unsigned int ret_freq = 0;
1338
1339         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1340                 return cpufreq_driver->get(cpu);
1341
1342         policy = cpufreq_cpu_get(cpu);
1343         if (policy) {
1344                 ret_freq = policy->cur;
1345                 cpufreq_cpu_put(policy);
1346         }
1347
1348         return ret_freq;
1349 }
1350 EXPORT_SYMBOL(cpufreq_quick_get);
1351
1352 /**
1353  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1354  * @cpu: CPU number
1355  *
1356  * Just return the max possible frequency for a given CPU.
1357  */
1358 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1359 {
1360         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1361         unsigned int ret_freq = 0;
1362
1363         if (policy) {
1364                 ret_freq = policy->max;
1365                 cpufreq_cpu_put(policy);
1366         }
1367
1368         return ret_freq;
1369 }
1370 EXPORT_SYMBOL(cpufreq_quick_get_max);
1371
1372 static unsigned int __cpufreq_get(unsigned int cpu)
1373 {
1374         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1375         unsigned int ret_freq = 0;
1376
1377         if (!cpufreq_driver->get)
1378                 return ret_freq;
1379
1380         ret_freq = cpufreq_driver->get(cpu);
1381
1382         if (ret_freq && policy->cur &&
1383                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1384                 /* verify no discrepancy between actual and
1385                                         saved value exists */
1386                 if (unlikely(ret_freq != policy->cur)) {
1387                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1388                         schedule_work(&policy->update);
1389                 }
1390         }
1391
1392         return ret_freq;
1393 }
1394
1395 /**
1396  * cpufreq_get - get the current CPU frequency (in kHz)
1397  * @cpu: CPU number
1398  *
1399  * Get the CPU current (static) CPU frequency
1400  */
1401 unsigned int cpufreq_get(unsigned int cpu)
1402 {
1403         unsigned int ret_freq = 0;
1404
1405         if (!down_read_trylock(&cpufreq_rwsem))
1406                 return 0;
1407
1408         if (unlikely(lock_policy_rwsem_read(cpu)))
1409                 goto out_policy;
1410
1411         ret_freq = __cpufreq_get(cpu);
1412
1413         unlock_policy_rwsem_read(cpu);
1414
1415 out_policy:
1416         up_read(&cpufreq_rwsem);
1417
1418         return ret_freq;
1419 }
1420 EXPORT_SYMBOL(cpufreq_get);
1421
1422 static struct subsys_interface cpufreq_interface = {
1423         .name           = "cpufreq",
1424         .subsys         = &cpu_subsys,
1425         .add_dev        = cpufreq_add_dev,
1426         .remove_dev     = cpufreq_remove_dev,
1427 };
1428
1429 /**
1430  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1431  *
1432  * This function is only executed for the boot processor.  The other CPUs
1433  * have been put offline by means of CPU hotplug.
1434  */
1435 static int cpufreq_bp_suspend(void)
1436 {
1437         int ret = 0;
1438
1439         int cpu = smp_processor_id();
1440         struct cpufreq_policy *policy;
1441
1442         pr_debug("suspending cpu %u\n", cpu);
1443
1444         /* If there's no policy for the boot CPU, we have nothing to do. */
1445         policy = cpufreq_cpu_get(cpu);
1446         if (!policy)
1447                 return 0;
1448
1449         if (cpufreq_driver->suspend) {
1450                 ret = cpufreq_driver->suspend(policy);
1451                 if (ret)
1452                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1453                                         "step on CPU %u\n", policy->cpu);
1454         }
1455
1456         cpufreq_cpu_put(policy);
1457         return ret;
1458 }
1459
1460 /**
1461  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1462  *
1463  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1464  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1465  *          restored. It will verify that the current freq is in sync with
1466  *          what we believe it to be. This is a bit later than when it
1467  *          should be, but nonethteless it's better than calling
1468  *          cpufreq_driver->get() here which might re-enable interrupts...
1469  *
1470  * This function is only executed for the boot CPU.  The other CPUs have not
1471  * been turned on yet.
1472  */
1473 static void cpufreq_bp_resume(void)
1474 {
1475         int ret = 0;
1476
1477         int cpu = smp_processor_id();
1478         struct cpufreq_policy *policy;
1479
1480         pr_debug("resuming cpu %u\n", cpu);
1481
1482         /* If there's no policy for the boot CPU, we have nothing to do. */
1483         policy = cpufreq_cpu_get(cpu);
1484         if (!policy)
1485                 return;
1486
1487         if (cpufreq_driver->resume) {
1488                 ret = cpufreq_driver->resume(policy);
1489                 if (ret) {
1490                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1491                                         "step on CPU %u\n", policy->cpu);
1492                         goto fail;
1493                 }
1494         }
1495
1496         schedule_work(&policy->update);
1497
1498 fail:
1499         cpufreq_cpu_put(policy);
1500 }
1501
1502 static struct syscore_ops cpufreq_syscore_ops = {
1503         .suspend        = cpufreq_bp_suspend,
1504         .resume         = cpufreq_bp_resume,
1505 };
1506
1507 /**
1508  *      cpufreq_get_current_driver - return current driver's name
1509  *
1510  *      Return the name string of the currently loaded cpufreq driver
1511  *      or NULL, if none.
1512  */
1513 const char *cpufreq_get_current_driver(void)
1514 {
1515         if (cpufreq_driver)
1516                 return cpufreq_driver->name;
1517
1518         return NULL;
1519 }
1520 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1521
1522 /*********************************************************************
1523  *                     NOTIFIER LISTS INTERFACE                      *
1524  *********************************************************************/
1525
1526 /**
1527  *      cpufreq_register_notifier - register a driver with cpufreq
1528  *      @nb: notifier function to register
1529  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1530  *
1531  *      Add a driver to one of two lists: either a list of drivers that
1532  *      are notified about clock rate changes (once before and once after
1533  *      the transition), or a list of drivers that are notified about
1534  *      changes in cpufreq policy.
1535  *
1536  *      This function may sleep, and has the same return conditions as
1537  *      blocking_notifier_chain_register.
1538  */
1539 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1540 {
1541         int ret;
1542
1543         if (cpufreq_disabled())
1544                 return -EINVAL;
1545
1546         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1547
1548         switch (list) {
1549         case CPUFREQ_TRANSITION_NOTIFIER:
1550                 ret = srcu_notifier_chain_register(
1551                                 &cpufreq_transition_notifier_list, nb);
1552                 break;
1553         case CPUFREQ_POLICY_NOTIFIER:
1554                 ret = blocking_notifier_chain_register(
1555                                 &cpufreq_policy_notifier_list, nb);
1556                 break;
1557         default:
1558                 ret = -EINVAL;
1559         }
1560
1561         return ret;
1562 }
1563 EXPORT_SYMBOL(cpufreq_register_notifier);
1564
1565 /**
1566  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1567  *      @nb: notifier block to be unregistered
1568  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1569  *
1570  *      Remove a driver from the CPU frequency notifier list.
1571  *
1572  *      This function may sleep, and has the same return conditions as
1573  *      blocking_notifier_chain_unregister.
1574  */
1575 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1576 {
1577         int ret;
1578
1579         if (cpufreq_disabled())
1580                 return -EINVAL;
1581
1582         switch (list) {
1583         case CPUFREQ_TRANSITION_NOTIFIER:
1584                 ret = srcu_notifier_chain_unregister(
1585                                 &cpufreq_transition_notifier_list, nb);
1586                 break;
1587         case CPUFREQ_POLICY_NOTIFIER:
1588                 ret = blocking_notifier_chain_unregister(
1589                                 &cpufreq_policy_notifier_list, nb);
1590                 break;
1591         default:
1592                 ret = -EINVAL;
1593         }
1594
1595         return ret;
1596 }
1597 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1598
1599
1600 /*********************************************************************
1601  *                              GOVERNORS                            *
1602  *********************************************************************/
1603
1604 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1605                             unsigned int target_freq,
1606                             unsigned int relation)
1607 {
1608         int retval = -EINVAL;
1609         unsigned int old_target_freq = target_freq;
1610
1611         if (cpufreq_disabled())
1612                 return -ENODEV;
1613         if (policy->transition_ongoing)
1614                 return -EBUSY;
1615
1616         /* Make sure that target_freq is within supported range */
1617         if (target_freq > policy->max)
1618                 target_freq = policy->max;
1619         if (target_freq < policy->min)
1620                 target_freq = policy->min;
1621
1622         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1623                         policy->cpu, target_freq, relation, old_target_freq);
1624
1625         if (target_freq == policy->cur)
1626                 return 0;
1627
1628         if (cpufreq_driver->target)
1629                 retval = cpufreq_driver->target(policy, target_freq, relation);
1630
1631         return retval;
1632 }
1633 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1634
1635 int cpufreq_driver_target(struct cpufreq_policy *policy,
1636                           unsigned int target_freq,
1637                           unsigned int relation)
1638 {
1639         int ret = -EINVAL;
1640
1641         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1642                 goto fail;
1643
1644         ret = __cpufreq_driver_target(policy, target_freq, relation);
1645
1646         unlock_policy_rwsem_write(policy->cpu);
1647
1648 fail:
1649         return ret;
1650 }
1651 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1652
1653 /*
1654  * when "event" is CPUFREQ_GOV_LIMITS
1655  */
1656
1657 static int __cpufreq_governor(struct cpufreq_policy *policy,
1658                                         unsigned int event)
1659 {
1660         int ret;
1661
1662         /* Only must be defined when default governor is known to have latency
1663            restrictions, like e.g. conservative or ondemand.
1664            That this is the case is already ensured in Kconfig
1665         */
1666 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1667         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1668 #else
1669         struct cpufreq_governor *gov = NULL;
1670 #endif
1671
1672         if (policy->governor->max_transition_latency &&
1673             policy->cpuinfo.transition_latency >
1674             policy->governor->max_transition_latency) {
1675                 if (!gov)
1676                         return -EINVAL;
1677                 else {
1678                         printk(KERN_WARNING "%s governor failed, too long"
1679                                " transition latency of HW, fallback"
1680                                " to %s governor\n",
1681                                policy->governor->name,
1682                                gov->name);
1683                         policy->governor = gov;
1684                 }
1685         }
1686
1687         if (event == CPUFREQ_GOV_POLICY_INIT)
1688                 if (!try_module_get(policy->governor->owner))
1689                         return -EINVAL;
1690
1691         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1692                                                 policy->cpu, event);
1693
1694         mutex_lock(&cpufreq_governor_lock);
1695         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1696             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1697                 mutex_unlock(&cpufreq_governor_lock);
1698                 return -EBUSY;
1699         }
1700
1701         if (event == CPUFREQ_GOV_STOP)
1702                 policy->governor_enabled = false;
1703         else if (event == CPUFREQ_GOV_START)
1704                 policy->governor_enabled = true;
1705
1706         mutex_unlock(&cpufreq_governor_lock);
1707
1708         ret = policy->governor->governor(policy, event);
1709
1710         if (!ret) {
1711                 if (event == CPUFREQ_GOV_POLICY_INIT)
1712                         policy->governor->initialized++;
1713                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1714                         policy->governor->initialized--;
1715         } else {
1716                 /* Restore original values */
1717                 mutex_lock(&cpufreq_governor_lock);
1718                 if (event == CPUFREQ_GOV_STOP)
1719                         policy->governor_enabled = true;
1720                 else if (event == CPUFREQ_GOV_START)
1721                         policy->governor_enabled = false;
1722                 mutex_unlock(&cpufreq_governor_lock);
1723         }
1724
1725         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1726                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1727                 module_put(policy->governor->owner);
1728
1729         return ret;
1730 }
1731
1732 int cpufreq_register_governor(struct cpufreq_governor *governor)
1733 {
1734         int err;
1735
1736         if (!governor)
1737                 return -EINVAL;
1738
1739         if (cpufreq_disabled())
1740                 return -ENODEV;
1741
1742         mutex_lock(&cpufreq_governor_mutex);
1743
1744         governor->initialized = 0;
1745         err = -EBUSY;
1746         if (__find_governor(governor->name) == NULL) {
1747                 err = 0;
1748                 list_add(&governor->governor_list, &cpufreq_governor_list);
1749         }
1750
1751         mutex_unlock(&cpufreq_governor_mutex);
1752         return err;
1753 }
1754 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1755
1756 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1757 {
1758 #ifdef CONFIG_HOTPLUG_CPU
1759         int cpu;
1760 #endif
1761
1762         if (!governor)
1763                 return;
1764
1765         if (cpufreq_disabled())
1766                 return;
1767
1768 #ifdef CONFIG_HOTPLUG_CPU
1769         for_each_present_cpu(cpu) {
1770                 if (cpu_online(cpu))
1771                         continue;
1772                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1773                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1774         }
1775 #endif
1776
1777         mutex_lock(&cpufreq_governor_mutex);
1778         list_del(&governor->governor_list);
1779         mutex_unlock(&cpufreq_governor_mutex);
1780         return;
1781 }
1782 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1783
1784
1785 /*********************************************************************
1786  *                          POLICY INTERFACE                         *
1787  *********************************************************************/
1788
1789 /**
1790  * cpufreq_get_policy - get the current cpufreq_policy
1791  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1792  *      is written
1793  *
1794  * Reads the current cpufreq policy.
1795  */
1796 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1797 {
1798         struct cpufreq_policy *cpu_policy;
1799         if (!policy)
1800                 return -EINVAL;
1801
1802         cpu_policy = cpufreq_cpu_get(cpu);
1803         if (!cpu_policy)
1804                 return -EINVAL;
1805
1806         memcpy(policy, cpu_policy, sizeof(*policy));
1807
1808         cpufreq_cpu_put(cpu_policy);
1809         return 0;
1810 }
1811 EXPORT_SYMBOL(cpufreq_get_policy);
1812
1813 /*
1814  * data   : current policy.
1815  * policy : policy to be set.
1816  */
1817 static int __cpufreq_set_policy(struct cpufreq_policy *policy,
1818                                 struct cpufreq_policy *new_policy)
1819 {
1820         int ret = 0, failed = 1;
1821
1822         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
1823                 new_policy->min, new_policy->max);
1824
1825         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
1826
1827         if (new_policy->min > policy->max || new_policy->max < policy->min) {
1828                 ret = -EINVAL;
1829                 goto error_out;
1830         }
1831
1832         /* verify the cpu speed can be set within this limit */
1833         ret = cpufreq_driver->verify(new_policy);
1834         if (ret)
1835                 goto error_out;
1836
1837         /* adjust if necessary - all reasons */
1838         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1839                         CPUFREQ_ADJUST, new_policy);
1840
1841         /* adjust if necessary - hardware incompatibility*/
1842         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1843                         CPUFREQ_INCOMPATIBLE, new_policy);
1844
1845         /*
1846          * verify the cpu speed can be set within this limit, which might be
1847          * different to the first one
1848          */
1849         ret = cpufreq_driver->verify(new_policy);
1850         if (ret)
1851                 goto error_out;
1852
1853         /* notification of the new policy */
1854         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1855                         CPUFREQ_NOTIFY, new_policy);
1856
1857         policy->min = new_policy->min;
1858         policy->max = new_policy->max;
1859
1860         pr_debug("new min and max freqs are %u - %u kHz\n",
1861                                         policy->min, policy->max);
1862
1863         if (cpufreq_driver->setpolicy) {
1864                 policy->policy = new_policy->policy;
1865                 pr_debug("setting range\n");
1866                 ret = cpufreq_driver->setpolicy(new_policy);
1867         } else {
1868                 if (new_policy->governor != policy->governor) {
1869                         /* save old, working values */
1870                         struct cpufreq_governor *old_gov = policy->governor;
1871
1872                         pr_debug("governor switch\n");
1873
1874                         /* end old governor */
1875                         if (policy->governor) {
1876                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1877                                 unlock_policy_rwsem_write(new_policy->cpu);
1878                                 __cpufreq_governor(policy,
1879                                                 CPUFREQ_GOV_POLICY_EXIT);
1880                                 lock_policy_rwsem_write(new_policy->cpu);
1881                         }
1882
1883                         /* start new governor */
1884                         policy->governor = new_policy->governor;
1885                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
1886                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
1887                                         failed = 0;
1888                                 } else {
1889                                         unlock_policy_rwsem_write(new_policy->cpu);
1890                                         __cpufreq_governor(policy,
1891                                                         CPUFREQ_GOV_POLICY_EXIT);
1892                                         lock_policy_rwsem_write(new_policy->cpu);
1893                                 }
1894                         }
1895
1896                         if (failed) {
1897                                 /* new governor failed, so re-start old one */
1898                                 pr_debug("starting governor %s failed\n",
1899                                                         policy->governor->name);
1900                                 if (old_gov) {
1901                                         policy->governor = old_gov;
1902                                         __cpufreq_governor(policy,
1903                                                         CPUFREQ_GOV_POLICY_INIT);
1904                                         __cpufreq_governor(policy,
1905                                                            CPUFREQ_GOV_START);
1906                                 }
1907                                 ret = -EINVAL;
1908                                 goto error_out;
1909                         }
1910                         /* might be a policy change, too, so fall through */
1911                 }
1912                 pr_debug("governor: change or update limits\n");
1913                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1914         }
1915
1916 error_out:
1917         return ret;
1918 }
1919
1920 /**
1921  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1922  *      @cpu: CPU which shall be re-evaluated
1923  *
1924  *      Useful for policy notifiers which have different necessities
1925  *      at different times.
1926  */
1927 int cpufreq_update_policy(unsigned int cpu)
1928 {
1929         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1930         struct cpufreq_policy new_policy;
1931         int ret;
1932
1933         if (!policy) {
1934                 ret = -ENODEV;
1935                 goto no_policy;
1936         }
1937
1938         if (unlikely(lock_policy_rwsem_write(cpu))) {
1939                 ret = -EINVAL;
1940                 goto fail;
1941         }
1942
1943         pr_debug("updating policy for CPU %u\n", cpu);
1944         memcpy(&new_policy, policy, sizeof(*policy));
1945         new_policy.min = policy->user_policy.min;
1946         new_policy.max = policy->user_policy.max;
1947         new_policy.policy = policy->user_policy.policy;
1948         new_policy.governor = policy->user_policy.governor;
1949
1950         /*
1951          * BIOS might change freq behind our back
1952          * -> ask driver for current freq and notify governors about a change
1953          */
1954         if (cpufreq_driver->get) {
1955                 new_policy.cur = cpufreq_driver->get(cpu);
1956                 if (!policy->cur) {
1957                         pr_debug("Driver did not initialize current freq");
1958                         policy->cur = new_policy.cur;
1959                 } else {
1960                         if (policy->cur != new_policy.cur && cpufreq_driver->target)
1961                                 cpufreq_out_of_sync(cpu, policy->cur,
1962                                                                 new_policy.cur);
1963                 }
1964         }
1965
1966         ret = __cpufreq_set_policy(policy, &new_policy);
1967
1968         unlock_policy_rwsem_write(cpu);
1969
1970 fail:
1971         cpufreq_cpu_put(policy);
1972 no_policy:
1973         return ret;
1974 }
1975 EXPORT_SYMBOL(cpufreq_update_policy);
1976
1977 static int cpufreq_cpu_callback(struct notifier_block *nfb,
1978                                         unsigned long action, void *hcpu)
1979 {
1980         unsigned int cpu = (unsigned long)hcpu;
1981         struct device *dev;
1982         bool frozen = false;
1983
1984         dev = get_cpu_device(cpu);
1985         if (dev) {
1986
1987                 if (action & CPU_TASKS_FROZEN)
1988                         frozen = true;
1989
1990                 switch (action & ~CPU_TASKS_FROZEN) {
1991                 case CPU_ONLINE:
1992                         __cpufreq_add_dev(dev, NULL, frozen);
1993                         cpufreq_update_policy(cpu);
1994                         break;
1995
1996                 case CPU_DOWN_PREPARE:
1997                         __cpufreq_remove_dev(dev, NULL, frozen);
1998                         break;
1999
2000                 case CPU_DOWN_FAILED:
2001                         __cpufreq_add_dev(dev, NULL, frozen);
2002                         break;
2003                 }
2004         }
2005         return NOTIFY_OK;
2006 }
2007
2008 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2009         .notifier_call = cpufreq_cpu_callback,
2010 };
2011
2012 /*********************************************************************
2013  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2014  *********************************************************************/
2015
2016 /**
2017  * cpufreq_register_driver - register a CPU Frequency driver
2018  * @driver_data: A struct cpufreq_driver containing the values#
2019  * submitted by the CPU Frequency driver.
2020  *
2021  * Registers a CPU Frequency driver to this core code. This code
2022  * returns zero on success, -EBUSY when another driver got here first
2023  * (and isn't unregistered in the meantime).
2024  *
2025  */
2026 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2027 {
2028         unsigned long flags;
2029         int ret;
2030
2031         if (cpufreq_disabled())
2032                 return -ENODEV;
2033
2034         if (!driver_data || !driver_data->verify || !driver_data->init ||
2035             ((!driver_data->setpolicy) && (!driver_data->target)))
2036                 return -EINVAL;
2037
2038         pr_debug("trying to register driver %s\n", driver_data->name);
2039
2040         if (driver_data->setpolicy)
2041                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2042
2043         write_lock_irqsave(&cpufreq_driver_lock, flags);
2044         if (cpufreq_driver) {
2045                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2046                 return -EBUSY;
2047         }
2048         cpufreq_driver = driver_data;
2049         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2050
2051         ret = subsys_interface_register(&cpufreq_interface);
2052         if (ret)
2053                 goto err_null_driver;
2054
2055         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2056                 int i;
2057                 ret = -ENODEV;
2058
2059                 /* check for at least one working CPU */
2060                 for (i = 0; i < nr_cpu_ids; i++)
2061                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2062                                 ret = 0;
2063                                 break;
2064                         }
2065
2066                 /* if all ->init() calls failed, unregister */
2067                 if (ret) {
2068                         pr_debug("no CPU initialized for driver %s\n",
2069                                                         driver_data->name);
2070                         goto err_if_unreg;
2071                 }
2072         }
2073
2074         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2075         pr_debug("driver %s up and running\n", driver_data->name);
2076
2077         return 0;
2078 err_if_unreg:
2079         subsys_interface_unregister(&cpufreq_interface);
2080 err_null_driver:
2081         write_lock_irqsave(&cpufreq_driver_lock, flags);
2082         cpufreq_driver = NULL;
2083         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2084         return ret;
2085 }
2086 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2087
2088 /**
2089  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2090  *
2091  * Unregister the current CPUFreq driver. Only call this if you have
2092  * the right to do so, i.e. if you have succeeded in initialising before!
2093  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2094  * currently not initialised.
2095  */
2096 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2097 {
2098         unsigned long flags;
2099
2100         if (!cpufreq_driver || (driver != cpufreq_driver))
2101                 return -EINVAL;
2102
2103         pr_debug("unregistering driver %s\n", driver->name);
2104
2105         subsys_interface_unregister(&cpufreq_interface);
2106         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2107
2108         down_write(&cpufreq_rwsem);
2109         write_lock_irqsave(&cpufreq_driver_lock, flags);
2110
2111         cpufreq_driver = NULL;
2112
2113         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2114         up_write(&cpufreq_rwsem);
2115
2116         return 0;
2117 }
2118 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2119
2120 static int __init cpufreq_core_init(void)
2121 {
2122         int cpu;
2123
2124         if (cpufreq_disabled())
2125                 return -ENODEV;
2126
2127         for_each_possible_cpu(cpu)
2128                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2129
2130         cpufreq_global_kobject = kobject_create();
2131         BUG_ON(!cpufreq_global_kobject);
2132         register_syscore_ops(&cpufreq_syscore_ops);
2133
2134         return 0;
2135 }
2136 core_initcall(cpufreq_core_init);