drivers/staging/lustre/lnet/libcfs/workitem.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2011, 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/workitem.c
  37  *
  38  * Author: Isaac Huang <isaac@clusterfs.com>
  39  *       Liang Zhen  <zhen.liang@sun.com>
  40  */
  41
  42 #define DEBUG_SUBSYSTEM S_LNET
  43
  44 #include "../../include/linux/libcfs/libcfs.h"
  45
  46 #define CFS_WS_NAME_LEN  16
  47
  48 struct cfs_wi_sched {
  49         /* chain on global list */
  50         struct list_head                ws_list;
  51         /** serialised workitems */
  52         spinlock_t              ws_lock;
  53         /** where schedulers sleep */
  54         wait_queue_head_t               ws_waitq;
  55         /** concurrent workitems */
  56         struct list_head                ws_runq;
  57         /**
  58          * rescheduled running-workitems, a workitem can be rescheduled
  59          * while running in wi_action(), but we don't to execute it again
  60          * unless it returns from wi_action(), so we put it on ws_rerunq
  61          * while rescheduling, and move it to runq after it returns
  62          * from wi_action()
  63          */
  64         struct list_head                ws_rerunq;
  65         /** CPT-table for this scheduler */
  66         struct cfs_cpt_table    *ws_cptab;
  67         /** CPT id for affinity */
  68         int                     ws_cpt;
  69         /** number of scheduled workitems */
  70         int                     ws_nscheduled;
  71         /** started scheduler thread, protected by cfs_wi_data::wi_glock */
  72         unsigned int            ws_nthreads:30;
  73         /** shutting down, protected by cfs_wi_data::wi_glock */
  74         unsigned int            ws_stopping:1;
  75         /** serialize starting thread, protected by cfs_wi_data::wi_glock */
  76         unsigned int            ws_starting:1;
  77         /** scheduler name */
  78         char                    ws_name[CFS_WS_NAME_LEN];
  79 };
  80
  81 static struct cfs_workitem_data {
  82         /** serialize */
  83         spinlock_t              wi_glock;
  84         /** list of all schedulers */
  85         struct list_head                wi_scheds;
  86         /** WI module is initialized */
  87         int                     wi_init;
  88         /** shutting down the whole WI module */
  89         int                     wi_stopping;
  90 } cfs_wi_data;
  91
  92 static inline int
  93 cfs_wi_sched_cansleep(struct cfs_wi_sched *sched)
  94 {
  95         spin_lock(&sched->ws_lock);
  96         if (sched->ws_stopping) {
  97                 spin_unlock(&sched->ws_lock);
  98                 return 0;
  99         }
 100
 101         if (!list_empty(&sched->ws_runq)) {
 102                 spin_unlock(&sched->ws_lock);
 103                 return 0;
 104         }
 105         spin_unlock(&sched->ws_lock);
 106         return 1;
 107 }
 108
 109 /* XXX:
 110  * 0. it only works when called from wi->wi_action.
 111  * 1. when it returns no one shall try to schedule the workitem.
 112  */
 113 void
 114 cfs_wi_exit(struct cfs_wi_sched *sched, struct cfs_workitem *wi)
 115 {
 116         LASSERT(!in_interrupt()); /* because we use plain spinlock */
 117         LASSERT(!sched->ws_stopping);
 118
 119         spin_lock(&sched->ws_lock);
 120
 121         LASSERT(wi->wi_running);
 122         if (wi->wi_scheduled) { /* cancel pending schedules */
 123                 LASSERT(!list_empty(&wi->wi_list));
 124                 list_del_init(&wi->wi_list);
 125
 126                 LASSERT(sched->ws_nscheduled > 0);
 127                 sched->ws_nscheduled--;
 128         }
 129
 130         LASSERT(list_empty(&wi->wi_list));
 131
 132         wi->wi_scheduled = 1; /* LBUG future schedule attempts */
 133         spin_unlock(&sched->ws_lock);
 134 }
 135 EXPORT_SYMBOL(cfs_wi_exit);
 136
 137 /**
 138  * cancel schedule request of workitem \a wi
 139  */
 140 int
 141 cfs_wi_deschedule(struct cfs_wi_sched *sched, struct cfs_workitem *wi)
 142 {
 143         int     rc;
 144
 145         LASSERT(!in_interrupt()); /* because we use plain spinlock */
 146         LASSERT(!sched->ws_stopping);
 147
 148         /*
 149          * return 0 if it's running already, otherwise return 1, which
 150          * means the workitem will not be scheduled and will not have
 151          * any race with wi_action.
 152          */
 153         spin_lock(&sched->ws_lock);
 154
 155         rc = !(wi->wi_running);
 156
 157         if (wi->wi_scheduled) { /* cancel pending schedules */
 158                 LASSERT(!list_empty(&wi->wi_list));
 159                 list_del_init(&wi->wi_list);
 160
 161                 LASSERT(sched->ws_nscheduled > 0);
 162                 sched->ws_nscheduled--;
 163
 164                 wi->wi_scheduled = 0;
 165         }
 166
 167         LASSERT(list_empty(&wi->wi_list));
 168
 169         spin_unlock(&sched->ws_lock);
 170         return rc;
 171 }
 172 EXPORT_SYMBOL(cfs_wi_deschedule);
 173
 174 /*
 175  * Workitem scheduled with (serial == 1) is strictly serialised not only with
 176  * itself, but also with others scheduled this way.
 177  *
 178  * Now there's only one static serialised queue, but in the future more might
 179  * be added, and even dynamic creation of serialised queues might be supported.
 180  */
 181 void
 182 cfs_wi_schedule(struct cfs_wi_sched *sched, struct cfs_workitem *wi)
 183 {
 184         LASSERT(!in_interrupt()); /* because we use plain spinlock */
 185         LASSERT(!sched->ws_stopping);
 186
 187         spin_lock(&sched->ws_lock);
 188
 189         if (!wi->wi_scheduled) {
 190                 LASSERT(list_empty(&wi->wi_list));
 191
 192                 wi->wi_scheduled = 1;
 193                 sched->ws_nscheduled++;
 194                 if (!wi->wi_running) {
 195                         list_add_tail(&wi->wi_list, &sched->ws_runq);
 196                         wake_up(&sched->ws_waitq);
 197                 } else {
 198                         list_add(&wi->wi_list, &sched->ws_rerunq);
 199                 }
 200         }
 201
 202         LASSERT(!list_empty(&wi->wi_list));
 203         spin_unlock(&sched->ws_lock);
 204 }
 205 EXPORT_SYMBOL(cfs_wi_schedule);
 206
 207 static int cfs_wi_scheduler(void *arg)
 208 {
 209         struct cfs_wi_sched     *sched = (struct cfs_wi_sched *)arg;
 210
 211         cfs_block_allsigs();
 212
 213         /* CPT affinity scheduler? */
 214         if (sched->ws_cptab)
 215                 if (cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt) != 0)
 216                         CWARN("Failed to bind %s on CPT %d\n",
 217                               sched->ws_name, sched->ws_cpt);
 218
 219         spin_lock(&cfs_wi_data.wi_glock);
 220
 221         LASSERT(sched->ws_starting == 1);
 222         sched->ws_starting--;
 223         sched->ws_nthreads++;
 224
 225         spin_unlock(&cfs_wi_data.wi_glock);
 226
 227         spin_lock(&sched->ws_lock);
 228
 229         while (!sched->ws_stopping) {
 230                 int          nloops = 0;
 231                 int          rc;
 232                 struct cfs_workitem *wi;
 233
 234                 while (!list_empty(&sched->ws_runq) &&
 235                        nloops < CFS_WI_RESCHED) {
 236                         wi = list_entry(sched->ws_runq.next,
 237                                         struct cfs_workitem, wi_list);
 238                         LASSERT(wi->wi_scheduled && !wi->wi_running);
 239
 240                         list_del_init(&wi->wi_list);
 241
 242                         LASSERT(sched->ws_nscheduled > 0);
 243                         sched->ws_nscheduled--;
 244
 245                         wi->wi_running   = 1;
 246                         wi->wi_scheduled = 0;
 247
 248                         spin_unlock(&sched->ws_lock);
 249                         nloops++;
 250
 251                         rc = (*wi->wi_action) (wi);
 252
 253                         spin_lock(&sched->ws_lock);
 254                         if (rc != 0) /* WI should be dead, even be freed! */
 255                                 continue;
 256
 257                         wi->wi_running = 0;
 258                         if (list_empty(&wi->wi_list))
 259                                 continue;
 260
 261                         LASSERT(wi->wi_scheduled);
 262                         /* wi is rescheduled, should be on rerunq now, we
 263                          * move it to runq so it can run action now
 264                          */
 265                         list_move_tail(&wi->wi_list, &sched->ws_runq);
 266                 }
 267
 268                 if (!list_empty(&sched->ws_runq)) {
 269                         spin_unlock(&sched->ws_lock);
 270                         /* don't sleep because some workitems still
 271                          * expect me to come back soon
 272                          */
 273                         cond_resched();
 274                         spin_lock(&sched->ws_lock);
 275                         continue;
 276                 }
 277
 278                 spin_unlock(&sched->ws_lock);
 279                 rc = wait_event_interruptible_exclusive(sched->ws_waitq,
 280                                                 !cfs_wi_sched_cansleep(sched));
 281                 spin_lock(&sched->ws_lock);
 282         }
 283
 284         spin_unlock(&sched->ws_lock);
 285
 286         spin_lock(&cfs_wi_data.wi_glock);
 287         sched->ws_nthreads--;
 288         spin_unlock(&cfs_wi_data.wi_glock);
 289
 290         return 0;
 291 }
 292
 293 void
 294 cfs_wi_sched_destroy(struct cfs_wi_sched *sched)
 295 {
 296         int     i;
 297
 298         LASSERT(cfs_wi_data.wi_init);
 299         LASSERT(!cfs_wi_data.wi_stopping);
 300
 301         spin_lock(&cfs_wi_data.wi_glock);
 302         if (sched->ws_stopping) {
 303                 CDEBUG(D_INFO, "%s is in progress of stopping\n",
 304                        sched->ws_name);
 305                 spin_unlock(&cfs_wi_data.wi_glock);
 306                 return;
 307         }
 308
 309         LASSERT(!list_empty(&sched->ws_list));
 310         sched->ws_stopping = 1;
 311
 312         spin_unlock(&cfs_wi_data.wi_glock);
 313
 314         i = 2;
 315         wake_up_all(&sched->ws_waitq);
 316
 317         spin_lock(&cfs_wi_data.wi_glock);
 318         while (sched->ws_nthreads > 0) {
 319                 CDEBUG(is_power_of_2(++i) ? D_WARNING : D_NET,
 320                        "waiting for %d threads of WI sched[%s] to terminate\n",
 321                        sched->ws_nthreads, sched->ws_name);
 322
 323                 spin_unlock(&cfs_wi_data.wi_glock);
 324                 set_current_state(TASK_UNINTERRUPTIBLE);
 325                 schedule_timeout(cfs_time_seconds(1) / 20);
 326                 spin_lock(&cfs_wi_data.wi_glock);
 327         }
 328
 329         list_del(&sched->ws_list);
 330
 331         spin_unlock(&cfs_wi_data.wi_glock);
 332         LASSERT(sched->ws_nscheduled == 0);
 333
 334         LIBCFS_FREE(sched, sizeof(*sched));
 335 }
 336 EXPORT_SYMBOL(cfs_wi_sched_destroy);
 337
 338 int
 339 cfs_wi_sched_create(char *name, struct cfs_cpt_table *cptab,
 340                     int cpt, int nthrs, struct cfs_wi_sched **sched_pp)
 341 {
 342         struct cfs_wi_sched     *sched;
 343         int                     rc;
 344
 345         LASSERT(cfs_wi_data.wi_init);
 346         LASSERT(!cfs_wi_data.wi_stopping);
 347         LASSERT(!cptab || cpt == CFS_CPT_ANY ||
 348                 (cpt >= 0 && cpt < cfs_cpt_number(cptab)));
 349
 350         LIBCFS_ALLOC(sched, sizeof(*sched));
 351         if (!sched)
 352                 return -ENOMEM;
 353
 354         if (strlen(name) > sizeof(sched->ws_name) - 1) {
 355                 LIBCFS_FREE(sched, sizeof(*sched));
 356                 return -E2BIG;
 357         }
 358         strncpy(sched->ws_name, name, sizeof(sched->ws_name));
 359
 360         sched->ws_cptab = cptab;
 361         sched->ws_cpt = cpt;
 362
 363         spin_lock_init(&sched->ws_lock);
 364         init_waitqueue_head(&sched->ws_waitq);
 365         INIT_LIST_HEAD(&sched->ws_runq);
 366         INIT_LIST_HEAD(&sched->ws_rerunq);
 367         INIT_LIST_HEAD(&sched->ws_list);
 368
 369         rc = 0;
 370         while (nthrs > 0)  {
 371                 char    name[16];
 372                 struct task_struct *task;
 373
 374                 spin_lock(&cfs_wi_data.wi_glock);
 375                 while (sched->ws_starting > 0) {
 376                         spin_unlock(&cfs_wi_data.wi_glock);
 377                         schedule();
 378                         spin_lock(&cfs_wi_data.wi_glock);
 379                 }
 380
 381                 sched->ws_starting++;
 382                 spin_unlock(&cfs_wi_data.wi_glock);
 383
 384                 if (sched->ws_cptab && sched->ws_cpt >= 0) {
 385                         snprintf(name, sizeof(name), "%s_%02d_%02u",
 386                                  sched->ws_name, sched->ws_cpt,
 387                                  sched->ws_nthreads);
 388                 } else {
 389                         snprintf(name, sizeof(name), "%s_%02u",
 390                                  sched->ws_name, sched->ws_nthreads);
 391                 }
 392
 393                 task = kthread_run(cfs_wi_scheduler, sched, "%s", name);
 394                 if (!IS_ERR(task)) {
 395                         nthrs--;
 396                         continue;
 397                 }
 398                 rc = PTR_ERR(task);
 399
 400                 CERROR("Failed to create thread for WI scheduler %s: %d\n",
 401                        name, rc);
 402
 403                 spin_lock(&cfs_wi_data.wi_glock);
 404
 405                 /* make up for cfs_wi_sched_destroy */
 406                 list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
 407                 sched->ws_starting--;
 408
 409                 spin_unlock(&cfs_wi_data.wi_glock);
 410
 411                 cfs_wi_sched_destroy(sched);
 412                 return rc;
 413         }
 414         spin_lock(&cfs_wi_data.wi_glock);
 415         list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
 416         spin_unlock(&cfs_wi_data.wi_glock);
 417
 418         *sched_pp = sched;
 419         return 0;
 420 }
 421 EXPORT_SYMBOL(cfs_wi_sched_create);
 422
 423 int
 424 cfs_wi_startup(void)
 425 {
 426         memset(&cfs_wi_data, 0, sizeof(cfs_wi_data));
 427
 428         spin_lock_init(&cfs_wi_data.wi_glock);
 429         INIT_LIST_HEAD(&cfs_wi_data.wi_scheds);
 430         cfs_wi_data.wi_init = 1;
 431
 432         return 0;
 433 }
 434
 435 void
 436 cfs_wi_shutdown(void)
 437 {
 438         struct cfs_wi_sched     *sched;
 439         struct cfs_wi_sched *temp;
 440
 441         spin_lock(&cfs_wi_data.wi_glock);
 442         cfs_wi_data.wi_stopping = 1;
 443         spin_unlock(&cfs_wi_data.wi_glock);
 444
 445         /* nobody should contend on this list */
 446         list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
 447                 sched->ws_stopping = 1;
 448                 wake_up_all(&sched->ws_waitq);
 449         }
 450
 451         list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
 452                 spin_lock(&cfs_wi_data.wi_glock);
 453
 454                 while (sched->ws_nthreads != 0) {
 455                         spin_unlock(&cfs_wi_data.wi_glock);
 456                         set_current_state(TASK_UNINTERRUPTIBLE);
 457                         schedule_timeout(cfs_time_seconds(1) / 20);
 458                         spin_lock(&cfs_wi_data.wi_glock);
 459                 }
 460                 spin_unlock(&cfs_wi_data.wi_glock);
 461         }
 462         list_for_each_entry_safe(sched, temp, &cfs_wi_data.wi_scheds, ws_list) {
 463                 list_del(&sched->ws_list);
 464                 LIBCFS_FREE(sched, sizeof(*sched));
 465         }
 466
 467         cfs_wi_data.wi_stopping = 0;
 468         cfs_wi_data.wi_init = 0;
 469 }