drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/slab.h>
  25 #include <linux/list.h>
  26 #include <linux/types.h>
  27 #include <linux/printk.h>
  28 #include <linux/bitops.h>
  29 #include <linux/sched.h>
  30 #include "kfd_priv.h"
  31 #include "kfd_device_queue_manager.h"
  32 #include "kfd_mqd_manager.h"
  33 #include "cik_regs.h"
  34 #include "kfd_kernel_queue.h"
  35
  36 /* Size of the per-pipe EOP queue */
  37 #define CIK_HPD_EOP_BYTES_LOG2 11
  38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  39
  40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  41                                         unsigned int pasid, unsigned int vmid);
  42
  43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
  44                                         struct queue *q,
  45                                         struct qcm_process_device *qpd);
  46
  47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  49
  50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
  51                                         struct queue *q,
  52                                         struct qcm_process_device *qpd);
  53
  54 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
  55                                 unsigned int sdma_queue_id);
  56
  57 static inline
  58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
  59 {
  60         if (type == KFD_QUEUE_TYPE_SDMA)
  61                 return KFD_MQD_TYPE_SDMA;
  62         return KFD_MQD_TYPE_CP;
  63 }
  64
  65 unsigned int get_first_pipe(struct device_queue_manager *dqm)
  66 {
  67         BUG_ON(!dqm || !dqm->dev);
  68         return dqm->dev->shared_resources.first_compute_pipe;
  69 }
  70
  71 unsigned int get_pipes_num(struct device_queue_manager *dqm)
  72 {
  73         BUG_ON(!dqm || !dqm->dev);
  74         return dqm->dev->shared_resources.compute_pipe_count;
  75 }
  76
  77 static inline unsigned int get_pipes_num_cpsch(void)
  78 {
  79         return PIPE_PER_ME_CP_SCHEDULING;
  80 }
  81
  82 void program_sh_mem_settings(struct device_queue_manager *dqm,
  83                                         struct qcm_process_device *qpd)
  84 {
  85         return dqm->dev->kfd2kgd->program_sh_mem_settings(
  86                                                 dqm->dev->kgd, qpd->vmid,
  87                                                 qpd->sh_mem_config,
  88                                                 qpd->sh_mem_ape1_base,
  89                                                 qpd->sh_mem_ape1_limit,
  90                                                 qpd->sh_mem_bases);
  91 }
  92
  93 static int allocate_vmid(struct device_queue_manager *dqm,
  94                         struct qcm_process_device *qpd,
  95                         struct queue *q)
  96 {
  97         int bit, allocated_vmid;
  98
  99         if (dqm->vmid_bitmap == 0)
 100                 return -ENOMEM;
 101
 102         bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
 103         clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 104
 105         /* Kaveri kfd vmid's starts from vmid 8 */
 106         allocated_vmid = bit + KFD_VMID_START_OFFSET;
 107         pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
 108         qpd->vmid = allocated_vmid;
 109         q->properties.vmid = allocated_vmid;
 110
 111         set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
 112         program_sh_mem_settings(dqm, qpd);
 113
 114         return 0;
 115 }
 116
 117 static void deallocate_vmid(struct device_queue_manager *dqm,
 118                                 struct qcm_process_device *qpd,
 119                                 struct queue *q)
 120 {
 121         int bit = qpd->vmid - KFD_VMID_START_OFFSET;
 122
 123         /* Release the vmid mapping */
 124         set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 125
 126         set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 127         qpd->vmid = 0;
 128         q->properties.vmid = 0;
 129 }
 130
 131 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 132                                 struct queue *q,
 133                                 struct qcm_process_device *qpd,
 134                                 int *allocated_vmid)
 135 {
 136         int retval;
 137
 138         BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
 139
 140         pr_debug("kfd: In func %s\n", __func__);
 141         print_queue(q);
 142
 143         mutex_lock(&dqm->lock);
 144
 145         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 146                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 147                                 dqm->total_queue_count);
 148                 mutex_unlock(&dqm->lock);
 149                 return -EPERM;
 150         }
 151
 152         if (list_empty(&qpd->queues_list)) {
 153                 retval = allocate_vmid(dqm, qpd, q);
 154                 if (retval != 0) {
 155                         mutex_unlock(&dqm->lock);
 156                         return retval;
 157                 }
 158         }
 159         *allocated_vmid = qpd->vmid;
 160         q->properties.vmid = qpd->vmid;
 161
 162         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 163                 retval = create_compute_queue_nocpsch(dqm, q, qpd);
 164         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 165                 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
 166
 167         if (retval != 0) {
 168                 if (list_empty(&qpd->queues_list)) {
 169                         deallocate_vmid(dqm, qpd, q);
 170                         *allocated_vmid = 0;
 171                 }
 172                 mutex_unlock(&dqm->lock);
 173                 return retval;
 174         }
 175
 176         list_add(&q->list, &qpd->queues_list);
 177         if (q->properties.is_active)
 178                 dqm->queue_count++;
 179
 180         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 181                 dqm->sdma_queue_count++;
 182
 183         /*
 184          * Unconditionally increment this counter, regardless of the queue's
 185          * type or whether the queue is active.
 186          */
 187         dqm->total_queue_count++;
 188         pr_debug("Total of %d queues are accountable so far\n",
 189                         dqm->total_queue_count);
 190
 191         mutex_unlock(&dqm->lock);
 192         return 0;
 193 }
 194
 195 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 196 {
 197         bool set;
 198         int pipe, bit, i;
 199
 200         set = false;
 201
 202         for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm);
 203                         pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) {
 204                 if (dqm->allocated_queues[pipe] != 0) {
 205                         bit = find_first_bit(
 206                                 (unsigned long *)&dqm->allocated_queues[pipe],
 207                                 QUEUES_PER_PIPE);
 208
 209                         clear_bit(bit,
 210                                 (unsigned long *)&dqm->allocated_queues[pipe]);
 211                         q->pipe = pipe;
 212                         q->queue = bit;
 213                         set = true;
 214                         break;
 215                 }
 216         }
 217
 218         if (set == false)
 219                 return -EBUSY;
 220
 221         pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
 222                                 __func__, q->pipe, q->queue);
 223         /* horizontal hqd allocation */
 224         dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
 225
 226         return 0;
 227 }
 228
 229 static inline void deallocate_hqd(struct device_queue_manager *dqm,
 230                                 struct queue *q)
 231 {
 232         set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
 233 }
 234
 235 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 236                                         struct queue *q,
 237                                         struct qcm_process_device *qpd)
 238 {
 239         int retval;
 240         struct mqd_manager *mqd;
 241
 242         BUG_ON(!dqm || !q || !qpd);
 243
 244         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 245         if (mqd == NULL)
 246                 return -ENOMEM;
 247
 248         retval = allocate_hqd(dqm, q);
 249         if (retval != 0)
 250                 return retval;
 251
 252         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 253                                 &q->gart_mqd_addr, &q->properties);
 254         if (retval != 0) {
 255                 deallocate_hqd(dqm, q);
 256                 return retval;
 257         }
 258
 259         pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
 260                         q->pipe,
 261                         q->queue);
 262
 263         retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
 264                         q->queue, (uint32_t __user *) q->properties.write_ptr);
 265         if (retval != 0) {
 266                 deallocate_hqd(dqm, q);
 267                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 268                 return retval;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 275                                 struct qcm_process_device *qpd,
 276                                 struct queue *q)
 277 {
 278         int retval;
 279         struct mqd_manager *mqd;
 280
 281         BUG_ON(!dqm || !q || !q->mqd || !qpd);
 282
 283         retval = 0;
 284
 285         pr_debug("kfd: In Func %s\n", __func__);
 286
 287         mutex_lock(&dqm->lock);
 288
 289         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 290                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 291                 if (mqd == NULL) {
 292                         retval = -ENOMEM;
 293                         goto out;
 294                 }
 295                 deallocate_hqd(dqm, q);
 296         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 297                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 298                 if (mqd == NULL) {
 299                         retval = -ENOMEM;
 300                         goto out;
 301                 }
 302                 dqm->sdma_queue_count--;
 303                 deallocate_sdma_queue(dqm, q->sdma_id);
 304         } else {
 305                 pr_debug("q->properties.type is invalid (%d)\n",
 306                                 q->properties.type);
 307                 retval = -EINVAL;
 308                 goto out;
 309         }
 310
 311         retval = mqd->destroy_mqd(mqd, q->mqd,
 312                                 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 313                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 314                                 q->pipe, q->queue);
 315
 316         if (retval != 0)
 317                 goto out;
 318
 319         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 320
 321         list_del(&q->list);
 322         if (list_empty(&qpd->queues_list))
 323                 deallocate_vmid(dqm, qpd, q);
 324         if (q->properties.is_active)
 325                 dqm->queue_count--;
 326
 327         /*
 328          * Unconditionally decrement this counter, regardless of the queue's
 329          * type
 330          */
 331         dqm->total_queue_count--;
 332         pr_debug("Total of %d queues are accountable so far\n",
 333                         dqm->total_queue_count);
 334
 335 out:
 336         mutex_unlock(&dqm->lock);
 337         return retval;
 338 }
 339
 340 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 341 {
 342         int retval;
 343         struct mqd_manager *mqd;
 344         bool prev_active = false;
 345
 346         BUG_ON(!dqm || !q || !q->mqd);
 347
 348         mutex_lock(&dqm->lock);
 349         mqd = dqm->ops.get_mqd_manager(dqm,
 350                         get_mqd_type_from_queue_type(q->properties.type));
 351         if (mqd == NULL) {
 352                 mutex_unlock(&dqm->lock);
 353                 return -ENOMEM;
 354         }
 355
 356         if (q->properties.is_active == true)
 357                 prev_active = true;
 358
 359         /*
 360          *
 361          * check active state vs. the previous state
 362          * and modify counter accordingly
 363          */
 364         retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
 365         if ((q->properties.is_active == true) && (prev_active == false))
 366                 dqm->queue_count++;
 367         else if ((q->properties.is_active == false) && (prev_active == true))
 368                 dqm->queue_count--;
 369
 370         if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
 371                 retval = execute_queues_cpsch(dqm, false);
 372
 373         mutex_unlock(&dqm->lock);
 374         return retval;
 375 }
 376
 377 static struct mqd_manager *get_mqd_manager_nocpsch(
 378                 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 379 {
 380         struct mqd_manager *mqd;
 381
 382         BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
 383
 384         pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
 385
 386         mqd = dqm->mqds[type];
 387         if (!mqd) {
 388                 mqd = mqd_manager_init(type, dqm->dev);
 389                 if (mqd == NULL)
 390                         pr_err("kfd: mqd manager is NULL");
 391                 dqm->mqds[type] = mqd;
 392         }
 393
 394         return mqd;
 395 }
 396
 397 static int register_process_nocpsch(struct device_queue_manager *dqm,
 398                                         struct qcm_process_device *qpd)
 399 {
 400         struct device_process_node *n;
 401         int retval;
 402
 403         BUG_ON(!dqm || !qpd);
 404
 405         pr_debug("kfd: In func %s\n", __func__);
 406
 407         n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
 408         if (!n)
 409                 return -ENOMEM;
 410
 411         n->qpd = qpd;
 412
 413         mutex_lock(&dqm->lock);
 414         list_add(&n->list, &dqm->queues);
 415
 416         retval = dqm->ops_asic_specific.register_process(dqm, qpd);
 417
 418         dqm->processes_count++;
 419
 420         mutex_unlock(&dqm->lock);
 421
 422         return retval;
 423 }
 424
 425 static int unregister_process_nocpsch(struct device_queue_manager *dqm,
 426                                         struct qcm_process_device *qpd)
 427 {
 428         int retval;
 429         struct device_process_node *cur, *next;
 430
 431         BUG_ON(!dqm || !qpd);
 432
 433         BUG_ON(!list_empty(&qpd->queues_list));
 434
 435         pr_debug("kfd: In func %s\n", __func__);
 436
 437         retval = 0;
 438         mutex_lock(&dqm->lock);
 439
 440         list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 441                 if (qpd == cur->qpd) {
 442                         list_del(&cur->list);
 443                         kfree(cur);
 444                         dqm->processes_count--;
 445                         goto out;
 446                 }
 447         }
 448         /* qpd not found in dqm list */
 449         retval = 1;
 450 out:
 451         mutex_unlock(&dqm->lock);
 452         return retval;
 453 }
 454
 455 static int
 456 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
 457                         unsigned int vmid)
 458 {
 459         uint32_t pasid_mapping;
 460
 461         pasid_mapping = (pasid == 0) ? 0 :
 462                 (uint32_t)pasid |
 463                 ATC_VMID_PASID_MAPPING_VALID;
 464
 465         return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
 466                                                 dqm->dev->kgd, pasid_mapping,
 467                                                 vmid);
 468 }
 469
 470 int init_pipelines(struct device_queue_manager *dqm,
 471                         unsigned int pipes_num, unsigned int first_pipe)
 472 {
 473         void *hpdptr;
 474         struct mqd_manager *mqd;
 475         unsigned int i, err, inx;
 476         uint64_t pipe_hpd_addr;
 477
 478         BUG_ON(!dqm || !dqm->dev);
 479
 480         pr_debug("kfd: In func %s\n", __func__);
 481
 482         /*
 483          * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
 484          * The driver never accesses this memory after zeroing it.
 485          * It doesn't even have to be saved/restored on suspend/resume
 486          * because it contains no data when there are no active queues.
 487          */
 488
 489         err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num,
 490                                         &dqm->pipeline_mem);
 491
 492         if (err) {
 493                 pr_err("kfd: error allocate vidmem num pipes: %d\n",
 494                         pipes_num);
 495                 return -ENOMEM;
 496         }
 497
 498         hpdptr = dqm->pipeline_mem->cpu_ptr;
 499         dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
 500
 501         memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
 502
 503         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 504         if (mqd == NULL) {
 505                 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 506                 return -ENOMEM;
 507         }
 508
 509         for (i = 0; i < pipes_num; i++) {
 510                 inx = i + first_pipe;
 511                 /*
 512                  * HPD buffer on GTT is allocated by amdkfd, no need to waste
 513                  * space in GTT for pipelines we don't initialize
 514                  */
 515                 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
 516                 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
 517                 /* = log2(bytes/4)-1 */
 518                 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx,
 519                                 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int init_scheduler(struct device_queue_manager *dqm)
 526 {
 527         int retval;
 528
 529         BUG_ON(!dqm);
 530
 531         pr_debug("kfd: In %s\n", __func__);
 532
 533         retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm));
 534         return retval;
 535 }
 536
 537 static int initialize_nocpsch(struct device_queue_manager *dqm)
 538 {
 539         int i;
 540
 541         BUG_ON(!dqm);
 542
 543         pr_debug("kfd: In func %s num of pipes: %d\n",
 544                         __func__, get_pipes_num(dqm));
 545
 546         mutex_init(&dqm->lock);
 547         INIT_LIST_HEAD(&dqm->queues);
 548         dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 549         dqm->sdma_queue_count = 0;
 550         dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
 551                                         sizeof(unsigned int), GFP_KERNEL);
 552         if (!dqm->allocated_queues) {
 553                 mutex_destroy(&dqm->lock);
 554                 return -ENOMEM;
 555         }
 556
 557         for (i = 0; i < get_pipes_num(dqm); i++)
 558                 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
 559
 560         dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
 561         dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 562
 563         init_scheduler(dqm);
 564         return 0;
 565 }
 566
 567 static void uninitialize_nocpsch(struct device_queue_manager *dqm)
 568 {
 569         int i;
 570
 571         BUG_ON(!dqm);
 572
 573         BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
 574
 575         kfree(dqm->allocated_queues);
 576         for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
 577                 kfree(dqm->mqds[i]);
 578         mutex_destroy(&dqm->lock);
 579         kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 580 }
 581
 582 static int start_nocpsch(struct device_queue_manager *dqm)
 583 {
 584         return 0;
 585 }
 586
 587 static int stop_nocpsch(struct device_queue_manager *dqm)
 588 {
 589         return 0;
 590 }
 591
 592 static int allocate_sdma_queue(struct device_queue_manager *dqm,
 593                                 unsigned int *sdma_queue_id)
 594 {
 595         int bit;
 596
 597         if (dqm->sdma_bitmap == 0)
 598                 return -ENOMEM;
 599
 600         bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
 601                                 CIK_SDMA_QUEUES);
 602
 603         clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
 604         *sdma_queue_id = bit;
 605
 606         return 0;
 607 }
 608
 609 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 610                                 unsigned int sdma_queue_id)
 611 {
 612         if (sdma_queue_id >= CIK_SDMA_QUEUES)
 613                 return;
 614         set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
 615 }
 616
 617 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 618                                 struct qcm_process_device *qpd)
 619 {
 620         uint32_t value = SDMA_ATC;
 621
 622         if (q->process->is_32bit_user_mode)
 623                 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd));
 624         else
 625                 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64(
 626                                                         qpd_to_pdd(qpd)));
 627         q->properties.sdma_vm_addr = value;
 628 }
 629
 630 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 631                                         struct queue *q,
 632                                         struct qcm_process_device *qpd)
 633 {
 634         struct mqd_manager *mqd;
 635         int retval;
 636
 637         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 638         if (!mqd)
 639                 return -ENOMEM;
 640
 641         retval = allocate_sdma_queue(dqm, &q->sdma_id);
 642         if (retval != 0)
 643                 return retval;
 644
 645         q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 646         q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
 647
 648         pr_debug("kfd: sdma id is:    %d\n", q->sdma_id);
 649         pr_debug("     sdma queue id: %d\n", q->properties.sdma_queue_id);
 650         pr_debug("     sdma engine id: %d\n", q->properties.sdma_engine_id);
 651
 652         init_sdma_vm(dqm, q, qpd);
 653         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 654                                 &q->gart_mqd_addr, &q->properties);
 655         if (retval != 0) {
 656                 deallocate_sdma_queue(dqm, q->sdma_id);
 657                 return retval;
 658         }
 659
 660         retval = mqd->load_mqd(mqd, q->mqd, 0,
 661                                 0, NULL);
 662         if (retval != 0) {
 663                 deallocate_sdma_queue(dqm, q->sdma_id);
 664                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 665                 return retval;
 666         }
 667
 668         return 0;
 669 }
 670
 671 /*
 672  * Device Queue Manager implementation for cp scheduler
 673  */
 674
 675 static int set_sched_resources(struct device_queue_manager *dqm)
 676 {
 677         struct scheduling_resources res;
 678         unsigned int queue_num, queue_mask;
 679
 680         BUG_ON(!dqm);
 681
 682         pr_debug("kfd: In func %s\n", __func__);
 683
 684         queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
 685         queue_mask = (1 << queue_num) - 1;
 686         res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
 687         res.vmid_mask <<= KFD_VMID_START_OFFSET;
 688         res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
 689         res.gws_mask = res.oac_mask = res.gds_heap_base =
 690                                                 res.gds_heap_size = 0;
 691
 692         pr_debug("kfd: scheduling resources:\n"
 693                         "      vmid mask: 0x%8X\n"
 694                         "      queue mask: 0x%8llX\n",
 695                         res.vmid_mask, res.queue_mask);
 696
 697         return pm_send_set_resources(&dqm->packets, &res);
 698 }
 699
 700 static int initialize_cpsch(struct device_queue_manager *dqm)
 701 {
 702         int retval;
 703
 704         BUG_ON(!dqm);
 705
 706         pr_debug("kfd: In func %s num of pipes: %d\n",
 707                         __func__, get_pipes_num_cpsch());
 708
 709         mutex_init(&dqm->lock);
 710         INIT_LIST_HEAD(&dqm->queues);
 711         dqm->queue_count = dqm->processes_count = 0;
 712         dqm->sdma_queue_count = 0;
 713         dqm->active_runlist = false;
 714         retval = dqm->ops_asic_specific.initialize(dqm);
 715         if (retval != 0)
 716                 goto fail_init_pipelines;
 717
 718         return 0;
 719
 720 fail_init_pipelines:
 721         mutex_destroy(&dqm->lock);
 722         return retval;
 723 }
 724
 725 static int start_cpsch(struct device_queue_manager *dqm)
 726 {
 727         struct device_process_node *node;
 728         int retval;
 729
 730         BUG_ON(!dqm);
 731
 732         retval = 0;
 733
 734         retval = pm_init(&dqm->packets, dqm);
 735         if (retval != 0)
 736                 goto fail_packet_manager_init;
 737
 738         retval = set_sched_resources(dqm);
 739         if (retval != 0)
 740                 goto fail_set_sched_resources;
 741
 742         pr_debug("kfd: allocating fence memory\n");
 743
 744         /* allocate fence memory on the gart */
 745         retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
 746                                         &dqm->fence_mem);
 747
 748         if (retval != 0)
 749                 goto fail_allocate_vidmem;
 750
 751         dqm->fence_addr = dqm->fence_mem->cpu_ptr;
 752         dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
 753         list_for_each_entry(node, &dqm->queues, list)
 754                 if (node->qpd->pqm->process && dqm->dev)
 755                         kfd_bind_process_to_device(dqm->dev,
 756                                                 node->qpd->pqm->process);
 757
 758         execute_queues_cpsch(dqm, true);
 759
 760         return 0;
 761 fail_allocate_vidmem:
 762 fail_set_sched_resources:
 763         pm_uninit(&dqm->packets);
 764 fail_packet_manager_init:
 765         return retval;
 766 }
 767
 768 static int stop_cpsch(struct device_queue_manager *dqm)
 769 {
 770         struct device_process_node *node;
 771         struct kfd_process_device *pdd;
 772
 773         BUG_ON(!dqm);
 774
 775         destroy_queues_cpsch(dqm, true);
 776
 777         list_for_each_entry(node, &dqm->queues, list) {
 778                 pdd = qpd_to_pdd(node->qpd);
 779                 pdd->bound = false;
 780         }
 781         kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
 782         pm_uninit(&dqm->packets);
 783
 784         return 0;
 785 }
 786
 787 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 788                                         struct kernel_queue *kq,
 789                                         struct qcm_process_device *qpd)
 790 {
 791         BUG_ON(!dqm || !kq || !qpd);
 792
 793         pr_debug("kfd: In func %s\n", __func__);
 794
 795         mutex_lock(&dqm->lock);
 796         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 797                 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
 798                                 dqm->total_queue_count);
 799                 mutex_unlock(&dqm->lock);
 800                 return -EPERM;
 801         }
 802
 803         /*
 804          * Unconditionally increment this counter, regardless of the queue's
 805          * type or whether the queue is active.
 806          */
 807         dqm->total_queue_count++;
 808         pr_debug("Total of %d queues are accountable so far\n",
 809                         dqm->total_queue_count);
 810
 811         list_add(&kq->list, &qpd->priv_queue_list);
 812         dqm->queue_count++;
 813         qpd->is_debug = true;
 814         execute_queues_cpsch(dqm, false);
 815         mutex_unlock(&dqm->lock);
 816
 817         return 0;
 818 }
 819
 820 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 821                                         struct kernel_queue *kq,
 822                                         struct qcm_process_device *qpd)
 823 {
 824         BUG_ON(!dqm || !kq);
 825
 826         pr_debug("kfd: In %s\n", __func__);
 827
 828         mutex_lock(&dqm->lock);
 829         destroy_queues_cpsch(dqm, false);
 830         list_del(&kq->list);
 831         dqm->queue_count--;
 832         qpd->is_debug = false;
 833         execute_queues_cpsch(dqm, false);
 834         /*
 835          * Unconditionally decrement this counter, regardless of the queue's
 836          * type.
 837          */
 838         dqm->total_queue_count--;
 839         pr_debug("Total of %d queues are accountable so far\n",
 840                         dqm->total_queue_count);
 841         mutex_unlock(&dqm->lock);
 842 }
 843
 844 static void select_sdma_engine_id(struct queue *q)
 845 {
 846         static int sdma_id;
 847
 848         q->sdma_id = sdma_id;
 849         sdma_id = (sdma_id + 1) % 2;
 850 }
 851
 852 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 853                         struct qcm_process_device *qpd, int *allocate_vmid)
 854 {
 855         int retval;
 856         struct mqd_manager *mqd;
 857
 858         BUG_ON(!dqm || !q || !qpd);
 859
 860         retval = 0;
 861
 862         if (allocate_vmid)
 863                 *allocate_vmid = 0;
 864
 865         mutex_lock(&dqm->lock);
 866
 867         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 868                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 869                                 dqm->total_queue_count);
 870                 retval = -EPERM;
 871                 goto out;
 872         }
 873
 874         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 875                 select_sdma_engine_id(q);
 876
 877         mqd = dqm->ops.get_mqd_manager(dqm,
 878                         get_mqd_type_from_queue_type(q->properties.type));
 879
 880         if (mqd == NULL) {
 881                 mutex_unlock(&dqm->lock);
 882                 return -ENOMEM;
 883         }
 884
 885         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 886                                 &q->gart_mqd_addr, &q->properties);
 887         if (retval != 0)
 888                 goto out;
 889
 890         list_add(&q->list, &qpd->queues_list);
 891         if (q->properties.is_active) {
 892                 dqm->queue_count++;
 893                 retval = execute_queues_cpsch(dqm, false);
 894         }
 895
 896         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 897                         dqm->sdma_queue_count++;
 898         /*
 899          * Unconditionally increment this counter, regardless of the queue's
 900          * type or whether the queue is active.
 901          */
 902         dqm->total_queue_count++;
 903
 904         pr_debug("Total of %d queues are accountable so far\n",
 905                         dqm->total_queue_count);
 906
 907 out:
 908         mutex_unlock(&dqm->lock);
 909         return retval;
 910 }
 911
 912 static int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 913                                 unsigned int fence_value,
 914                                 unsigned long timeout)
 915 {
 916         BUG_ON(!fence_addr);
 917         timeout += jiffies;
 918
 919         while (*fence_addr != fence_value) {
 920                 if (time_after(jiffies, timeout)) {
 921                         pr_err("kfd: qcm fence wait loop timeout expired\n");
 922                         return -ETIME;
 923                 }
 924                 schedule();
 925         }
 926
 927         return 0;
 928 }
 929
 930 static int destroy_sdma_queues(struct device_queue_manager *dqm,
 931                                 unsigned int sdma_engine)
 932 {
 933         return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
 934                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false,
 935                         sdma_engine);
 936 }
 937
 938 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 939 {
 940         int retval;
 941
 942         BUG_ON(!dqm);
 943
 944         retval = 0;
 945
 946         if (lock)
 947                 mutex_lock(&dqm->lock);
 948         if (dqm->active_runlist == false)
 949                 goto out;
 950
 951         pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
 952                 dqm->sdma_queue_count);
 953
 954         if (dqm->sdma_queue_count > 0) {
 955                 destroy_sdma_queues(dqm, 0);
 956                 destroy_sdma_queues(dqm, 1);
 957         }
 958
 959         retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
 960                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
 961         if (retval != 0)
 962                 goto out;
 963
 964         *dqm->fence_addr = KFD_FENCE_INIT;
 965         pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
 966                                 KFD_FENCE_COMPLETED);
 967         /* should be timed out */
 968         amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 969                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
 970         pm_release_ib(&dqm->packets);
 971         dqm->active_runlist = false;
 972
 973 out:
 974         if (lock)
 975                 mutex_unlock(&dqm->lock);
 976         return retval;
 977 }
 978
 979 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 980 {
 981         int retval;
 982
 983         BUG_ON(!dqm);
 984
 985         if (lock)
 986                 mutex_lock(&dqm->lock);
 987
 988         retval = destroy_queues_cpsch(dqm, false);
 989         if (retval != 0) {
 990                 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
 991                 goto out;
 992         }
 993
 994         if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
 995                 retval = 0;
 996                 goto out;
 997         }
 998
 999         if (dqm->active_runlist) {
1000                 retval = 0;
1001                 goto out;
1002         }
1003
1004         retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1005         if (retval != 0) {
1006                 pr_err("kfd: failed to execute runlist");
1007                 goto out;
1008         }
1009         dqm->active_runlist = true;
1010
1011 out:
1012         if (lock)
1013                 mutex_unlock(&dqm->lock);
1014         return retval;
1015 }
1016
1017 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1018                                 struct qcm_process_device *qpd,
1019                                 struct queue *q)
1020 {
1021         int retval;
1022         struct mqd_manager *mqd;
1023
1024         BUG_ON(!dqm || !qpd || !q);
1025
1026         retval = 0;
1027
1028         /* remove queue from list to prevent rescheduling after preemption */
1029         mutex_lock(&dqm->lock);
1030         mqd = dqm->ops.get_mqd_manager(dqm,
1031                         get_mqd_type_from_queue_type(q->properties.type));
1032         if (!mqd) {
1033                 retval = -ENOMEM;
1034                 goto failed;
1035         }
1036
1037         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1038                 dqm->sdma_queue_count--;
1039
1040         list_del(&q->list);
1041         if (q->properties.is_active)
1042                 dqm->queue_count--;
1043
1044         execute_queues_cpsch(dqm, false);
1045
1046         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1047
1048         /*
1049          * Unconditionally decrement this counter, regardless of the queue's
1050          * type
1051          */
1052         dqm->total_queue_count--;
1053         pr_debug("Total of %d queues are accountable so far\n",
1054                         dqm->total_queue_count);
1055
1056         mutex_unlock(&dqm->lock);
1057
1058         return 0;
1059
1060 failed:
1061         mutex_unlock(&dqm->lock);
1062         return retval;
1063 }
1064
1065 /*
1066  * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1067  * stay in user mode.
1068  */
1069 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1070 /* APE1 limit is inclusive and 64K aligned. */
1071 #define APE1_LIMIT_ALIGNMENT 0xFFFF
1072
1073 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1074                                    struct qcm_process_device *qpd,
1075                                    enum cache_policy default_policy,
1076                                    enum cache_policy alternate_policy,
1077                                    void __user *alternate_aperture_base,
1078                                    uint64_t alternate_aperture_size)
1079 {
1080         bool retval;
1081
1082         pr_debug("kfd: In func %s\n", __func__);
1083
1084         mutex_lock(&dqm->lock);
1085
1086         if (alternate_aperture_size == 0) {
1087                 /* base > limit disables APE1 */
1088                 qpd->sh_mem_ape1_base = 1;
1089                 qpd->sh_mem_ape1_limit = 0;
1090         } else {
1091                 /*
1092                  * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1093                  *                      SH_MEM_APE1_BASE[31:0], 0x0000 }
1094                  * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1095                  *                      SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1096                  * Verify that the base and size parameters can be
1097                  * represented in this format and convert them.
1098                  * Additionally restrict APE1 to user-mode addresses.
1099                  */
1100
1101                 uint64_t base = (uintptr_t)alternate_aperture_base;
1102                 uint64_t limit = base + alternate_aperture_size - 1;
1103
1104                 if (limit <= base)
1105                         goto out;
1106
1107                 if ((base & APE1_FIXED_BITS_MASK) != 0)
1108                         goto out;
1109
1110                 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1111                         goto out;
1112
1113                 qpd->sh_mem_ape1_base = base >> 16;
1114                 qpd->sh_mem_ape1_limit = limit >> 16;
1115         }
1116
1117         retval = dqm->ops_asic_specific.set_cache_memory_policy(
1118                         dqm,
1119                         qpd,
1120                         default_policy,
1121                         alternate_policy,
1122                         alternate_aperture_base,
1123                         alternate_aperture_size);
1124
1125         if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1126                 program_sh_mem_settings(dqm, qpd);
1127
1128         pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1129                 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1130                 qpd->sh_mem_ape1_limit);
1131
1132         mutex_unlock(&dqm->lock);
1133         return retval;
1134
1135 out:
1136         mutex_unlock(&dqm->lock);
1137         return false;
1138 }
1139
1140 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1141 {
1142         struct device_queue_manager *dqm;
1143
1144         BUG_ON(!dev);
1145
1146         pr_debug("kfd: loading device queue manager\n");
1147
1148         dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1149         if (!dqm)
1150                 return NULL;
1151
1152         dqm->dev = dev;
1153         switch (sched_policy) {
1154         case KFD_SCHED_POLICY_HWS:
1155         case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1156                 /* initialize dqm for cp scheduling */
1157                 dqm->ops.create_queue = create_queue_cpsch;
1158                 dqm->ops.initialize = initialize_cpsch;
1159                 dqm->ops.start = start_cpsch;
1160                 dqm->ops.stop = stop_cpsch;
1161                 dqm->ops.destroy_queue = destroy_queue_cpsch;
1162                 dqm->ops.update_queue = update_queue;
1163                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1164                 dqm->ops.register_process = register_process_nocpsch;
1165                 dqm->ops.unregister_process = unregister_process_nocpsch;
1166                 dqm->ops.uninitialize = uninitialize_nocpsch;
1167                 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1168                 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1169                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1170                 break;
1171         case KFD_SCHED_POLICY_NO_HWS:
1172                 /* initialize dqm for no cp scheduling */
1173                 dqm->ops.start = start_nocpsch;
1174                 dqm->ops.stop = stop_nocpsch;
1175                 dqm->ops.create_queue = create_queue_nocpsch;
1176                 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1177                 dqm->ops.update_queue = update_queue;
1178                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1179                 dqm->ops.register_process = register_process_nocpsch;
1180                 dqm->ops.unregister_process = unregister_process_nocpsch;
1181                 dqm->ops.initialize = initialize_nocpsch;
1182                 dqm->ops.uninitialize = uninitialize_nocpsch;
1183                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1184                 break;
1185         default:
1186                 BUG();
1187                 break;
1188         }
1189
1190         switch (dev->device_info->asic_family) {
1191         case CHIP_CARRIZO:
1192                 device_queue_manager_init_vi(&dqm->ops_asic_specific);
1193                 break;
1194
1195         case CHIP_KAVERI:
1196                 device_queue_manager_init_cik(&dqm->ops_asic_specific);
1197                 break;
1198         }
1199
1200         if (dqm->ops.initialize(dqm) != 0) {
1201                 kfree(dqm);
1202                 return NULL;
1203         }
1204
1205         return dqm;
1206 }
1207
1208 void device_queue_manager_uninit(struct device_queue_manager *dqm)
1209 {
1210         BUG_ON(!dqm);
1211
1212         dqm->ops.uninitialize(dqm);
1213         kfree(dqm);
1214 }