drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 #define CREATE_TRACE_POINTS
  31 #include "gpu_sched_trace.h"
  32
  33 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
  34 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
  35
  36 struct kmem_cache *sched_fence_slab;
  37 atomic_t sched_fence_slab_ref = ATOMIC_INIT(0);
  38
  39 /* Initialize a given run queue struct */
  40 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  41 {
  42         spin_lock_init(&rq->lock);
  43         INIT_LIST_HEAD(&rq->entities);
  44         rq->current_entity = NULL;
  45 }
  46
  47 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  48                                     struct amd_sched_entity *entity)
  49 {
  50         if (!list_empty(&entity->list))
  51                 return;
  52         spin_lock(&rq->lock);
  53         list_add_tail(&entity->list, &rq->entities);
  54         spin_unlock(&rq->lock);
  55 }
  56
  57 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  58                                        struct amd_sched_entity *entity)
  59 {
  60         if (list_empty(&entity->list))
  61                 return;
  62         spin_lock(&rq->lock);
  63         list_del_init(&entity->list);
  64         if (rq->current_entity == entity)
  65                 rq->current_entity = NULL;
  66         spin_unlock(&rq->lock);
  67 }
  68
  69 /**
  70  * Select an entity which could provide a job to run
  71  *
  72  * @rq          The run queue to check.
  73  *
  74  * Try to find a ready entity, returns NULL if none found.
  75  */
  76 static struct amd_sched_entity *
  77 amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  78 {
  79         struct amd_sched_entity *entity;
  80
  81         spin_lock(&rq->lock);
  82
  83         entity = rq->current_entity;
  84         if (entity) {
  85                 list_for_each_entry_continue(entity, &rq->entities, list) {
  86                         if (amd_sched_entity_is_ready(entity)) {
  87                                 rq->current_entity = entity;
  88                                 spin_unlock(&rq->lock);
  89                                 return entity;
  90                         }
  91                 }
  92         }
  93
  94         list_for_each_entry(entity, &rq->entities, list) {
  95
  96                 if (amd_sched_entity_is_ready(entity)) {
  97                         rq->current_entity = entity;
  98                         spin_unlock(&rq->lock);
  99                         return entity;
 100                 }
 101
 102                 if (entity == rq->current_entity)
 103                         break;
 104         }
 105
 106         spin_unlock(&rq->lock);
 107
 108         return NULL;
 109 }
 110
 111 /**
 112  * Init a context entity used by scheduler when submit to HW ring.
 113  *
 114  * @sched       The pointer to the scheduler
 115  * @entity      The pointer to a valid amd_sched_entity
 116  * @rq          The run queue this entity belongs
 117  * @kernel      If this is an entity for the kernel
 118  * @jobs        The max number of jobs in the job queue
 119  *
 120  * return 0 if succeed. negative error code on failure
 121 */
 122 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 123                           struct amd_sched_entity *entity,
 124                           struct amd_sched_rq *rq,
 125                           uint32_t jobs)
 126 {
 127         int r;
 128
 129         if (!(sched && entity && rq))
 130                 return -EINVAL;
 131
 132         memset(entity, 0, sizeof(struct amd_sched_entity));
 133         INIT_LIST_HEAD(&entity->list);
 134         entity->rq = rq;
 135         entity->sched = sched;
 136
 137         spin_lock_init(&entity->queue_lock);
 138         r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
 139         if (r)
 140                 return r;
 141
 142         atomic_set(&entity->fence_seq, 0);
 143         entity->fence_context = fence_context_alloc(2);
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * Query if entity is initialized
 150  *
 151  * @sched       Pointer to scheduler instance
 152  * @entity      The pointer to a valid scheduler entity
 153  *
 154  * return true if entity is initialized, false otherwise
 155 */
 156 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
 157                                             struct amd_sched_entity *entity)
 158 {
 159         return entity->sched == sched &&
 160                 entity->rq != NULL;
 161 }
 162
 163 /**
 164  * Check if entity is idle
 165  *
 166  * @entity      The pointer to a valid scheduler entity
 167  *
 168  * Return true if entity don't has any unscheduled jobs.
 169  */
 170 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 171 {
 172         rmb();
 173         if (kfifo_is_empty(&entity->job_queue))
 174                 return true;
 175
 176         return false;
 177 }
 178
 179 /**
 180  * Check if entity is ready
 181  *
 182  * @entity      The pointer to a valid scheduler entity
 183  *
 184  * Return true if entity could provide a job.
 185  */
 186 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
 187 {
 188         if (kfifo_is_empty(&entity->job_queue))
 189                 return false;
 190
 191         if (ACCESS_ONCE(entity->dependency))
 192                 return false;
 193
 194         return true;
 195 }
 196
 197 /**
 198  * Destroy a context entity
 199  *
 200  * @sched       Pointer to scheduler instance
 201  * @entity      The pointer to a valid scheduler entity
 202  *
 203  * Cleanup and free the allocated resources.
 204  */
 205 void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 206                            struct amd_sched_entity *entity)
 207 {
 208         struct amd_sched_rq *rq = entity->rq;
 209
 210         if (!amd_sched_entity_is_initialized(sched, entity))
 211                 return;
 212
 213         /**
 214          * The client will not queue more IBs during this fini, consume existing
 215          * queued IBs
 216         */
 217         wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
 218
 219         amd_sched_rq_remove_entity(rq, entity);
 220         kfifo_free(&entity->job_queue);
 221 }
 222
 223 static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
 224 {
 225         struct amd_sched_entity *entity =
 226                 container_of(cb, struct amd_sched_entity, cb);
 227         entity->dependency = NULL;
 228         fence_put(f);
 229         amd_sched_wakeup(entity->sched);
 230 }
 231
 232 static void amd_sched_entity_clear_dep(struct fence *f, struct fence_cb *cb)
 233 {
 234         struct amd_sched_entity *entity =
 235                 container_of(cb, struct amd_sched_entity, cb);
 236         entity->dependency = NULL;
 237         fence_put(f);
 238 }
 239
 240 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 241 {
 242         struct amd_gpu_scheduler *sched = entity->sched;
 243         struct fence * fence = entity->dependency;
 244         struct amd_sched_fence *s_fence;
 245
 246         if (fence->context == entity->fence_context) {
 247                 /* We can ignore fences from ourself */
 248                 fence_put(entity->dependency);
 249                 return false;
 250         }
 251
 252         s_fence = to_amd_sched_fence(fence);
 253         if (s_fence && s_fence->sched == sched) {
 254
 255                 /*
 256                  * Fence is from the same scheduler, only need to wait for
 257                  * it to be scheduled
 258                  */
 259                 fence = fence_get(&s_fence->scheduled);
 260                 fence_put(entity->dependency);
 261                 entity->dependency = fence;
 262                 if (!fence_add_callback(fence, &entity->cb,
 263                                         amd_sched_entity_clear_dep))
 264                         return true;
 265
 266                 /* Ignore it when it is already scheduled */
 267                 fence_put(fence);
 268                 return false;
 269         }
 270
 271         if (!fence_add_callback(entity->dependency, &entity->cb,
 272                                 amd_sched_entity_wakeup))
 273                 return true;
 274
 275         fence_put(entity->dependency);
 276         return false;
 277 }
 278
 279 static struct amd_sched_job *
 280 amd_sched_entity_pop_job(struct amd_sched_entity *entity)
 281 {
 282         struct amd_gpu_scheduler *sched = entity->sched;
 283         struct amd_sched_job *sched_job;
 284
 285         if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
 286                 return NULL;
 287
 288         while ((entity->dependency = sched->ops->dependency(sched_job)))
 289                 if (amd_sched_entity_add_dependency_cb(entity))
 290                         return NULL;
 291
 292         return sched_job;
 293 }
 294
 295 /**
 296  * Helper to submit a job to the job queue
 297  *
 298  * @sched_job           The pointer to job required to submit
 299  *
 300  * Returns true if we could submit the job.
 301  */
 302 static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 303 {
 304         struct amd_gpu_scheduler *sched = sched_job->sched;
 305         struct amd_sched_entity *entity = sched_job->s_entity;
 306         bool added, first = false;
 307
 308         spin_lock(&entity->queue_lock);
 309         added = kfifo_in(&entity->job_queue, &sched_job,
 310                         sizeof(sched_job)) == sizeof(sched_job);
 311
 312         if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
 313                 first = true;
 314
 315         spin_unlock(&entity->queue_lock);
 316
 317         /* first job wakes up scheduler */
 318         if (first) {
 319                 /* Add the entity to the run queue */
 320                 amd_sched_rq_add_entity(entity->rq, entity);
 321                 amd_sched_wakeup(sched);
 322         }
 323         return added;
 324 }
 325
 326 /* job_finish is called after hw fence signaled, and
 327  * the job had already been deleted from ring_mirror_list
 328  */
 329 static void amd_sched_job_finish(struct work_struct *work)
 330 {
 331         struct amd_sched_job *s_job = container_of(work, struct amd_sched_job,
 332                                                    finish_work);
 333         struct amd_gpu_scheduler *sched = s_job->sched;
 334         unsigned long flags;
 335
 336         /* remove job from ring_mirror_list */
 337         spin_lock_irqsave(&sched->job_list_lock, flags);
 338         list_del_init(&s_job->node);
 339         if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
 340                 struct amd_sched_job *next;
 341
 342                 spin_unlock_irqrestore(&sched->job_list_lock, flags);
 343                 cancel_delayed_work_sync(&s_job->work_tdr);
 344                 spin_lock_irqsave(&sched->job_list_lock, flags);
 345
 346                 /* queue TDR for next job */
 347                 next = list_first_entry_or_null(&sched->ring_mirror_list,
 348                                                 struct amd_sched_job, node);
 349
 350                 if (next)
 351                         schedule_delayed_work(&next->work_tdr, sched->timeout);
 352         }
 353         spin_unlock_irqrestore(&sched->job_list_lock, flags);
 354         sched->ops->free_job(s_job);
 355 }
 356
 357 static void amd_sched_job_finish_cb(struct fence *f, struct fence_cb *cb)
 358 {
 359         struct amd_sched_job *job = container_of(cb, struct amd_sched_job,
 360                                                  finish_cb);
 361         schedule_work(&job->finish_work);
 362 }
 363
 364 static void amd_sched_job_begin(struct amd_sched_job *s_job)
 365 {
 366         struct amd_gpu_scheduler *sched = s_job->sched;
 367         unsigned long flags;
 368
 369         spin_lock_irqsave(&sched->job_list_lock, flags);
 370         list_add_tail(&s_job->node, &sched->ring_mirror_list);
 371         if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
 372             list_first_entry_or_null(&sched->ring_mirror_list,
 373                                      struct amd_sched_job, node) == s_job)
 374                 schedule_delayed_work(&s_job->work_tdr, sched->timeout);
 375         spin_unlock_irqrestore(&sched->job_list_lock, flags);
 376 }
 377
 378 static void amd_sched_job_timedout(struct work_struct *work)
 379 {
 380         struct amd_sched_job *job = container_of(work, struct amd_sched_job,
 381                                                  work_tdr.work);
 382
 383         job->sched->ops->timedout_job(job);
 384 }
 385
 386 /**
 387  * Submit a job to the job queue
 388  *
 389  * @sched_job           The pointer to job required to submit
 390  *
 391  * Returns 0 for success, negative error code otherwise.
 392  */
 393 void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 394 {
 395         struct amd_sched_entity *entity = sched_job->s_entity;
 396
 397         trace_amd_sched_job(sched_job);
 398         fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb,
 399                            amd_sched_job_finish_cb);
 400         wait_event(entity->sched->job_scheduled,
 401                    amd_sched_entity_in(sched_job));
 402 }
 403
 404 /* init a sched_job with basic field */
 405 int amd_sched_job_init(struct amd_sched_job *job,
 406                        struct amd_gpu_scheduler *sched,
 407                        struct amd_sched_entity *entity,
 408                        void *owner, struct fence **fence)
 409 {
 410         job->sched = sched;
 411         job->s_entity = entity;
 412         job->s_fence = amd_sched_fence_create(entity, owner);
 413         if (!job->s_fence)
 414                 return -ENOMEM;
 415
 416         INIT_WORK(&job->finish_work, amd_sched_job_finish);
 417         INIT_LIST_HEAD(&job->node);
 418         INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout);
 419
 420         if (fence)
 421                 *fence = &job->s_fence->finished;
 422         return 0;
 423 }
 424
 425 /**
 426  * Return ture if we can push more jobs to the hw.
 427  */
 428 static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 429 {
 430         return atomic_read(&sched->hw_rq_count) <
 431                 sched->hw_submission_limit;
 432 }
 433
 434 /**
 435  * Wake up the scheduler when it is ready
 436  */
 437 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 438 {
 439         if (amd_sched_ready(sched))
 440                 wake_up_interruptible(&sched->wake_up_worker);
 441 }
 442
 443 /**
 444  * Select next entity to process
 445 */
 446 static struct amd_sched_entity *
 447 amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 448 {
 449         struct amd_sched_entity *entity;
 450         int i;
 451
 452         if (!amd_sched_ready(sched))
 453                 return NULL;
 454
 455         /* Kernel run queue has higher priority than normal run queue*/
 456         for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) {
 457                 entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
 458                 if (entity)
 459                         break;
 460         }
 461
 462         return entity;
 463 }
 464
 465 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 466 {
 467         struct amd_sched_fence *s_fence =
 468                 container_of(cb, struct amd_sched_fence, cb);
 469         struct amd_gpu_scheduler *sched = s_fence->sched;
 470
 471         atomic_dec(&sched->hw_rq_count);
 472         amd_sched_fence_finished(s_fence);
 473
 474         trace_amd_sched_process_job(s_fence);
 475         fence_put(&s_fence->finished);
 476         wake_up_interruptible(&sched->wake_up_worker);
 477 }
 478
 479 static bool amd_sched_blocked(struct amd_gpu_scheduler *sched)
 480 {
 481         if (kthread_should_park()) {
 482                 kthread_parkme();
 483                 return true;
 484         }
 485
 486         return false;
 487 }
 488
 489 static int amd_sched_main(void *param)
 490 {
 491         struct sched_param sparam = {.sched_priority = 1};
 492         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 493         int r, count;
 494
 495         sched_setscheduler(current, SCHED_FIFO, &sparam);
 496
 497         while (!kthread_should_stop()) {
 498                 struct amd_sched_entity *entity = NULL;
 499                 struct amd_sched_fence *s_fence;
 500                 struct amd_sched_job *sched_job;
 501                 struct fence *fence;
 502
 503                 wait_event_interruptible(sched->wake_up_worker,
 504                                          (!amd_sched_blocked(sched) &&
 505                                           (entity = amd_sched_select_entity(sched))) ||
 506                                          kthread_should_stop());
 507
 508                 if (!entity)
 509                         continue;
 510
 511                 sched_job = amd_sched_entity_pop_job(entity);
 512                 if (!sched_job)
 513                         continue;
 514
 515                 s_fence = sched_job->s_fence;
 516
 517                 atomic_inc(&sched->hw_rq_count);
 518                 amd_sched_job_begin(sched_job);
 519
 520                 fence = sched->ops->run_job(sched_job);
 521                 amd_sched_fence_scheduled(s_fence);
 522                 if (fence) {
 523                         r = fence_add_callback(fence, &s_fence->cb,
 524                                                amd_sched_process_job);
 525                         if (r == -ENOENT)
 526                                 amd_sched_process_job(fence, &s_fence->cb);
 527                         else if (r)
 528                                 DRM_ERROR("fence add callback failed (%d)\n",
 529                                           r);
 530                         fence_put(fence);
 531                 } else {
 532                         DRM_ERROR("Failed to run job!\n");
 533                         amd_sched_process_job(NULL, &s_fence->cb);
 534                 }
 535
 536                 count = kfifo_out(&entity->job_queue, &sched_job,
 537                                 sizeof(sched_job));
 538                 WARN_ON(count != sizeof(sched_job));
 539                 wake_up(&sched->job_scheduled);
 540         }
 541         return 0;
 542 }
 543
 544 /**
 545  * Init a gpu scheduler instance
 546  *
 547  * @sched               The pointer to the scheduler
 548  * @ops                 The backend operations for this scheduler.
 549  * @hw_submissions      Number of hw submissions to do.
 550  * @name                Name used for debugging
 551  *
 552  * Return 0 on success, otherwise error code.
 553 */
 554 int amd_sched_init(struct amd_gpu_scheduler *sched,
 555                    const struct amd_sched_backend_ops *ops,
 556                    unsigned hw_submission, long timeout, const char *name)
 557 {
 558         int i;
 559         sched->ops = ops;
 560         sched->hw_submission_limit = hw_submission;
 561         sched->name = name;
 562         sched->timeout = timeout;
 563         for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++)
 564                 amd_sched_rq_init(&sched->sched_rq[i]);
 565
 566         init_waitqueue_head(&sched->wake_up_worker);
 567         init_waitqueue_head(&sched->job_scheduled);
 568         INIT_LIST_HEAD(&sched->ring_mirror_list);
 569         spin_lock_init(&sched->job_list_lock);
 570         atomic_set(&sched->hw_rq_count, 0);
 571         if (atomic_inc_return(&sched_fence_slab_ref) == 1) {
 572                 sched_fence_slab = kmem_cache_create(
 573                         "amd_sched_fence", sizeof(struct amd_sched_fence), 0,
 574                         SLAB_HWCACHE_ALIGN, NULL);
 575                 if (!sched_fence_slab)
 576                         return -ENOMEM;
 577         }
 578
 579         /* Each scheduler will run on a seperate kernel thread */
 580         sched->thread = kthread_run(amd_sched_main, sched, sched->name);
 581         if (IS_ERR(sched->thread)) {
 582                 DRM_ERROR("Failed to create scheduler for %s.\n", name);
 583                 return PTR_ERR(sched->thread);
 584         }
 585
 586         return 0;
 587 }
 588
 589 /**
 590  * Destroy a gpu scheduler
 591  *
 592  * @sched       The pointer to the scheduler
 593  */
 594 void amd_sched_fini(struct amd_gpu_scheduler *sched)
 595 {
 596         if (sched->thread)
 597                 kthread_stop(sched->thread);
 598         if (atomic_dec_and_test(&sched_fence_slab_ref))
 599                 kmem_cache_destroy(sched_fence_slab);
 600 }