kernel/torture.c

   1 /*
   2  * Common functions for in-kernel torture tests.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, you can access it online at
  16  * http://www.gnu.org/licenses/gpl-2.0.html.
  17  *
  18  * Copyright (C) IBM Corporation, 2014
  19  *
  20  * Author: Paul E. McKenney <paulmck@us.ibm.com>
  21  *      Based on kernel/rcu/torture.c.
  22  */
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/kthread.h>
  28 #include <linux/err.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/smp.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/sched.h>
  33 #include <linux/atomic.h>
  34 #include <linux/bitops.h>
  35 #include <linux/completion.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/percpu.h>
  38 #include <linux/notifier.h>
  39 #include <linux/reboot.h>
  40 #include <linux/freezer.h>
  41 #include <linux/cpu.h>
  42 #include <linux/delay.h>
  43 #include <linux/stat.h>
  44 #include <linux/slab.h>
  45 #include <linux/trace_clock.h>
  46 #include <asm/byteorder.h>
  47 #include <linux/torture.h>
  48
  49 MODULE_LICENSE("GPL");
  50 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
  51
  52 static char *torture_type;
  53 static bool verbose;
  54
  55 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
  56 #define FULLSTOP_DONTSTOP 0     /* Normal operation. */
  57 #define FULLSTOP_SHUTDOWN 1     /* System shutdown with torture running. */
  58 #define FULLSTOP_RMMOD    2     /* Normal rmmod of torture. */
  59 static int fullstop = FULLSTOP_RMMOD;
  60 static DEFINE_MUTEX(fullstop_mutex);
  61
  62 #ifdef CONFIG_HOTPLUG_CPU
  63
  64 /*
  65  * Variables for online-offline handling.  Only present if CPU hotplug
  66  * is enabled, otherwise does nothing.
  67  */
  68
  69 static struct task_struct *onoff_task;
  70 static long onoff_holdoff;
  71 static long onoff_interval;
  72 static long n_offline_attempts;
  73 static long n_offline_successes;
  74 static unsigned long sum_offline;
  75 static int min_offline = -1;
  76 static int max_offline;
  77 static long n_online_attempts;
  78 static long n_online_successes;
  79 static unsigned long sum_online;
  80 static int min_online = -1;
  81 static int max_online;
  82
  83 /*
  84  * Execute random CPU-hotplug operations at the interval specified
  85  * by the onoff_interval.
  86  */
  87 static int
  88 torture_onoff(void *arg)
  89 {
  90         int cpu;
  91         unsigned long delta;
  92         int maxcpu = -1;
  93         DEFINE_TORTURE_RANDOM(rand);
  94         int ret;
  95         unsigned long starttime;
  96
  97         VERBOSE_TOROUT_STRING("torture_onoff task started");
  98         for_each_online_cpu(cpu)
  99                 maxcpu = cpu;
 100         WARN_ON(maxcpu < 0);
 101         if (onoff_holdoff > 0) {
 102                 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
 103                 schedule_timeout_interruptible(onoff_holdoff);
 104                 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
 105         }
 106         while (!torture_must_stop()) {
 107                 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
 108                 if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) {
 109                         if (verbose)
 110                                 pr_alert("%s" TORTURE_FLAG
 111                                          "torture_onoff task: offlining %d\n",
 112                                          torture_type, cpu);
 113                         starttime = jiffies;
 114                         n_offline_attempts++;
 115                         ret = cpu_down(cpu);
 116                         if (ret) {
 117                                 if (verbose)
 118                                         pr_alert("%s" TORTURE_FLAG
 119                                                  "torture_onoff task: offline %d failed: errno %d\n",
 120                                                  torture_type, cpu, ret);
 121                         } else {
 122                                 if (verbose)
 123                                         pr_alert("%s" TORTURE_FLAG
 124                                                  "torture_onoff task: offlined %d\n",
 125                                                  torture_type, cpu);
 126                                 n_offline_successes++;
 127                                 delta = jiffies - starttime;
 128                                 sum_offline += delta;
 129                                 if (min_offline < 0) {
 130                                         min_offline = delta;
 131                                         max_offline = delta;
 132                                 }
 133                                 if (min_offline > delta)
 134                                         min_offline = delta;
 135                                 if (max_offline < delta)
 136                                         max_offline = delta;
 137                         }
 138                 } else if (cpu_is_hotpluggable(cpu)) {
 139                         if (verbose)
 140                                 pr_alert("%s" TORTURE_FLAG
 141                                          "torture_onoff task: onlining %d\n",
 142                                          torture_type, cpu);
 143                         starttime = jiffies;
 144                         n_online_attempts++;
 145                         ret = cpu_up(cpu);
 146                         if (ret) {
 147                                 if (verbose)
 148                                         pr_alert("%s" TORTURE_FLAG
 149                                                  "torture_onoff task: online %d failed: errno %d\n",
 150                                                  torture_type, cpu, ret);
 151                         } else {
 152                                 if (verbose)
 153                                         pr_alert("%s" TORTURE_FLAG
 154                                                  "torture_onoff task: onlined %d\n",
 155                                                  torture_type, cpu);
 156                                 n_online_successes++;
 157                                 delta = jiffies - starttime;
 158                                 sum_online += delta;
 159                                 if (min_online < 0) {
 160                                         min_online = delta;
 161                                         max_online = delta;
 162                                 }
 163                                 if (min_online > delta)
 164                                         min_online = delta;
 165                                 if (max_online < delta)
 166                                         max_online = delta;
 167                         }
 168                 }
 169                 schedule_timeout_interruptible(onoff_interval);
 170         }
 171         VERBOSE_TOROUT_STRING("torture_onoff task stopping");
 172         return 0;
 173 }
 174
 175 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 176
 177 /*
 178  * Initiate online-offline handling.
 179  */
 180 int torture_onoff_init(long ooholdoff, long oointerval)
 181 {
 182 #ifdef CONFIG_HOTPLUG_CPU
 183         int ret;
 184
 185         onoff_holdoff = ooholdoff;
 186         onoff_interval = oointerval;
 187         if (onoff_interval <= 0)
 188                 return 0;
 189         onoff_task = kthread_run(torture_onoff, NULL, "torture_onoff");
 190         if (IS_ERR(onoff_task)) {
 191                 ret = PTR_ERR(onoff_task);
 192                 onoff_task = NULL;
 193                 return ret;
 194         }
 195         torture_shuffle_task_register(onoff_task);
 196 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 197         return 0;
 198 }
 199 EXPORT_SYMBOL_GPL(torture_onoff_init);
 200
 201 /*
 202  * Clean up after online/offline testing.
 203  */
 204 static void torture_onoff_cleanup(void)
 205 {
 206 #ifdef CONFIG_HOTPLUG_CPU
 207         if (onoff_task == NULL)
 208                 return;
 209         VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
 210         kthread_stop(onoff_task);
 211         onoff_task = NULL;
 212 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 213 }
 214 EXPORT_SYMBOL_GPL(torture_onoff_cleanup);
 215
 216 /*
 217  * Print online/offline testing statistics.
 218  */
 219 char *torture_onoff_stats(char *page)
 220 {
 221 #ifdef CONFIG_HOTPLUG_CPU
 222         page += sprintf(page,
 223                        "onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
 224                        n_online_successes, n_online_attempts,
 225                        n_offline_successes, n_offline_attempts,
 226                        min_online, max_online,
 227                        min_offline, max_offline,
 228                        sum_online, sum_offline, HZ);
 229 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 230         return page;
 231 }
 232 EXPORT_SYMBOL_GPL(torture_onoff_stats);
 233
 234 /*
 235  * Were all the online/offline operations successful?
 236  */
 237 bool torture_onoff_failures(void)
 238 {
 239 #ifdef CONFIG_HOTPLUG_CPU
 240         return n_online_successes != n_online_attempts ||
 241                n_offline_successes != n_offline_attempts;
 242 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 243         return false;
 244 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 245 }
 246 EXPORT_SYMBOL_GPL(torture_onoff_failures);
 247
 248 #define TORTURE_RANDOM_MULT     39916801  /* prime */
 249 #define TORTURE_RANDOM_ADD      479001701 /* prime */
 250 #define TORTURE_RANDOM_REFRESH  10000
 251
 252 /*
 253  * Crude but fast random-number generator.  Uses a linear congruential
 254  * generator, with occasional help from cpu_clock().
 255  */
 256 unsigned long
 257 torture_random(struct torture_random_state *trsp)
 258 {
 259         if (--trsp->trs_count < 0) {
 260                 trsp->trs_state += (unsigned long)local_clock();
 261                 trsp->trs_count = TORTURE_RANDOM_REFRESH;
 262         }
 263         trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
 264                 TORTURE_RANDOM_ADD;
 265         return swahw32(trsp->trs_state);
 266 }
 267 EXPORT_SYMBOL_GPL(torture_random);
 268
 269 /*
 270  * Variables for shuffling.  The idea is to ensure that each CPU stays
 271  * idle for an extended period to test interactions with dyntick idle,
 272  * as well as interactions with any per-CPU varibles.
 273  */
 274 struct shuffle_task {
 275         struct list_head st_l;
 276         struct task_struct *st_t;
 277 };
 278
 279 static long shuffle_interval;   /* In jiffies. */
 280 static struct task_struct *shuffler_task;
 281 static cpumask_var_t shuffle_tmp_mask;
 282 static int shuffle_idle_cpu;    /* Force all torture tasks off this CPU */
 283 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
 284 static DEFINE_MUTEX(shuffle_task_mutex);
 285
 286 /*
 287  * Register a task to be shuffled.  If there is no memory, just splat
 288  * and don't bother registering.
 289  */
 290 void torture_shuffle_task_register(struct task_struct *tp)
 291 {
 292         struct shuffle_task *stp;
 293
 294         if (WARN_ON_ONCE(tp == NULL))
 295                 return;
 296         stp = kmalloc(sizeof(*stp), GFP_KERNEL);
 297         if (WARN_ON_ONCE(stp == NULL))
 298                 return;
 299         stp->st_t = tp;
 300         mutex_lock(&shuffle_task_mutex);
 301         list_add(&stp->st_l, &shuffle_task_list);
 302         mutex_unlock(&shuffle_task_mutex);
 303 }
 304 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
 305
 306 /*
 307  * Unregister all tasks, for example, at the end of the torture run.
 308  */
 309 static void torture_shuffle_task_unregister_all(void)
 310 {
 311         struct shuffle_task *stp;
 312         struct shuffle_task *p;
 313
 314         mutex_lock(&shuffle_task_mutex);
 315         list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
 316                 list_del(&stp->st_l);
 317                 kfree(stp);
 318         }
 319         mutex_unlock(&shuffle_task_mutex);
 320 }
 321
 322 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
 323  * A special case is when shuffle_idle_cpu = -1, in which case we allow
 324  * the tasks to run on all CPUs.
 325  */
 326 static void torture_shuffle_tasks(void)
 327 {
 328         struct shuffle_task *stp;
 329
 330         cpumask_setall(shuffle_tmp_mask);
 331         get_online_cpus();
 332
 333         /* No point in shuffling if there is only one online CPU (ex: UP) */
 334         if (num_online_cpus() == 1) {
 335                 put_online_cpus();
 336                 return;
 337         }
 338
 339         /* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
 340         shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
 341         if (shuffle_idle_cpu >= nr_cpu_ids)
 342                 shuffle_idle_cpu = -1;
 343         if (shuffle_idle_cpu != -1) {
 344                 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
 345                 if (cpumask_empty(shuffle_tmp_mask)) {
 346                         put_online_cpus();
 347                         return;
 348                 }
 349         }
 350
 351         mutex_lock(&shuffle_task_mutex);
 352         list_for_each_entry(stp, &shuffle_task_list, st_l)
 353                 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
 354         mutex_unlock(&shuffle_task_mutex);
 355
 356         put_online_cpus();
 357 }
 358
 359 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 360  * system to become idle at a time and cut off its timer ticks. This is meant
 361  * to test the support for such tickless idle CPU in RCU.
 362  */
 363 static int torture_shuffle(void *arg)
 364 {
 365         VERBOSE_TOROUT_STRING("torture_shuffle task started");
 366         do {
 367                 schedule_timeout_interruptible(shuffle_interval);
 368                 torture_shuffle_tasks();
 369                 torture_shutdown_absorb("torture_shuffle");
 370         } while (!torture_must_stop());
 371         VERBOSE_TOROUT_STRING("torture_shuffle task stopping");
 372         return 0;
 373 }
 374
 375 /*
 376  * Start the shuffler, with shuffint in jiffies.
 377  */
 378 int torture_shuffle_init(long shuffint)
 379 {
 380         int ret;
 381
 382         shuffle_interval = shuffint;
 383
 384         shuffle_idle_cpu = -1;
 385
 386         if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
 387                 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask");
 388                 return -ENOMEM;
 389         }
 390
 391         /* Create the shuffler thread */
 392         shuffler_task = kthread_run(torture_shuffle, NULL, "torture_shuffle");
 393         if (IS_ERR(shuffler_task)) {
 394                 ret = PTR_ERR(shuffler_task);
 395                 free_cpumask_var(shuffle_tmp_mask);
 396                 VERBOSE_TOROUT_ERRSTRING("Failed to create shuffler");
 397                 shuffler_task = NULL;
 398                 return ret;
 399         }
 400         torture_shuffle_task_register(shuffler_task);
 401         return 0;
 402 }
 403 EXPORT_SYMBOL_GPL(torture_shuffle_init);
 404
 405 /*
 406  * Stop the shuffling.
 407  */
 408 static void torture_shuffle_cleanup(void)
 409 {
 410         torture_shuffle_task_unregister_all();
 411         if (shuffler_task) {
 412                 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
 413                 kthread_stop(shuffler_task);
 414                 free_cpumask_var(shuffle_tmp_mask);
 415         }
 416         shuffler_task = NULL;
 417 }
 418 EXPORT_SYMBOL_GPL(torture_shuffle_cleanup);
 419
 420 /*
 421  * Absorb kthreads into a kernel function that won't return, so that
 422  * they won't ever access module text or data again.
 423  */
 424 void torture_shutdown_absorb(const char *title)
 425 {
 426         while (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 427                 pr_notice("torture thread %s parking due to system shutdown\n",
 428                           title);
 429                 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 430         }
 431 }
 432 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
 433
 434 /*
 435  * Detect and respond to a system shutdown.
 436  */
 437 static int torture_shutdown_notify(struct notifier_block *unused1,
 438                                    unsigned long unused2, void *unused3)
 439 {
 440         mutex_lock(&fullstop_mutex);
 441         if (fullstop == FULLSTOP_DONTSTOP)
 442                 fullstop = FULLSTOP_SHUTDOWN;
 443         else
 444                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 445         mutex_unlock(&fullstop_mutex);
 446         return NOTIFY_DONE;
 447 }
 448
 449 static struct notifier_block torture_shutdown_nb = {
 450         .notifier_call = torture_shutdown_notify,
 451 };
 452
 453 /*
 454  * Initialize torture module.  Please note that this is -not- invoked via
 455  * the usual module_init() mechanism, but rather by an explicit call from
 456  * the client torture module.  This call must be paired with a later
 457  * torture_init_end().
 458  */
 459 void __init torture_init_begin(char *ttype, bool v)
 460 {
 461         mutex_lock(&fullstop_mutex);
 462         torture_type = ttype;
 463         verbose = v;
 464         fullstop = FULLSTOP_DONTSTOP;
 465
 466 }
 467 EXPORT_SYMBOL_GPL(torture_init_begin);
 468
 469 /*
 470  * Tell the torture module that initialization is complete.
 471  */
 472 void __init torture_init_end(void)
 473 {
 474         mutex_unlock(&fullstop_mutex);
 475         register_reboot_notifier(&torture_shutdown_nb);
 476 }
 477 EXPORT_SYMBOL_GPL(torture_init_end);
 478
 479 /*
 480  * Clean up torture module.  Please note that this is -not- invoked via
 481  * the usual module_exit() mechanism, but rather by an explicit call from
 482  * the client torture module.  Returns true if a race with system shutdown
 483  * is detected.
 484  *
 485  * This must be called before the caller starts shutting down its own
 486  * kthreads.
 487  */
 488 bool torture_cleanup(void)
 489 {
 490         mutex_lock(&fullstop_mutex);
 491         if (fullstop == FULLSTOP_SHUTDOWN) {
 492                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 493                 mutex_unlock(&fullstop_mutex);
 494                 schedule_timeout_uninterruptible(10);
 495                 return true;
 496         }
 497         fullstop = FULLSTOP_RMMOD;
 498         mutex_unlock(&fullstop_mutex);
 499         unregister_reboot_notifier(&torture_shutdown_nb);
 500         torture_shuffle_cleanup();
 501         torture_onoff_cleanup();
 502         return false;
 503 }
 504 EXPORT_SYMBOL_GPL(torture_cleanup);
 505
 506 /*
 507  * Is it time for the current torture test to stop?
 508  */
 509 bool torture_must_stop(void)
 510 {
 511         return torture_must_stop_irq() || kthread_should_stop();
 512 }
 513 EXPORT_SYMBOL_GPL(torture_must_stop);
 514
 515 /*
 516  * Is it time for the current torture test to stop?  This is the irq-safe
 517  * version, hence no check for kthread_should_stop().
 518  */
 519 bool torture_must_stop_irq(void)
 520 {
 521         return fullstop != FULLSTOP_DONTSTOP;
 522 }
 523 EXPORT_SYMBOL_GPL(torture_must_stop_irq);