drivers/gpu/drm/msm/msm_gpu.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "msm_gpu.h"
  19 #include "msm_gem.h"
  20 #include "msm_mmu.h"
  21 #include "msm_fence.h"
  22
  23
  24 /*
  25  * Power Management:
  26  */
  27
  28 #ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
  29 #include <mach/board.h>
  30 static void bs_init(struct msm_gpu *gpu)
  31 {
  32         if (gpu->bus_scale_table) {
  33                 gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table);
  34                 DBG("bus scale client: %08x", gpu->bsc);
  35         }
  36 }
  37
  38 static void bs_fini(struct msm_gpu *gpu)
  39 {
  40         if (gpu->bsc) {
  41                 msm_bus_scale_unregister_client(gpu->bsc);
  42                 gpu->bsc = 0;
  43         }
  44 }
  45
  46 static void bs_set(struct msm_gpu *gpu, int idx)
  47 {
  48         if (gpu->bsc) {
  49                 DBG("set bus scaling: %d", idx);
  50                 msm_bus_scale_client_update_request(gpu->bsc, idx);
  51         }
  52 }
  53 #else
  54 static void bs_init(struct msm_gpu *gpu) {}
  55 static void bs_fini(struct msm_gpu *gpu) {}
  56 static void bs_set(struct msm_gpu *gpu, int idx) {}
  57 #endif
  58
  59 static int enable_pwrrail(struct msm_gpu *gpu)
  60 {
  61         struct drm_device *dev = gpu->dev;
  62         int ret = 0;
  63
  64         if (gpu->gpu_reg) {
  65                 ret = regulator_enable(gpu->gpu_reg);
  66                 if (ret) {
  67                         dev_err(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);
  68                         return ret;
  69                 }
  70         }
  71
  72         if (gpu->gpu_cx) {
  73                 ret = regulator_enable(gpu->gpu_cx);
  74                 if (ret) {
  75                         dev_err(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);
  76                         return ret;
  77                 }
  78         }
  79
  80         return 0;
  81 }
  82
  83 static int disable_pwrrail(struct msm_gpu *gpu)
  84 {
  85         if (gpu->gpu_cx)
  86                 regulator_disable(gpu->gpu_cx);
  87         if (gpu->gpu_reg)
  88                 regulator_disable(gpu->gpu_reg);
  89         return 0;
  90 }
  91
  92 static int enable_clk(struct msm_gpu *gpu)
  93 {
  94         struct clk *rate_clk = NULL;
  95         int i;
  96
  97         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
  98         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
  99                 if (gpu->grp_clks[i]) {
 100                         clk_prepare(gpu->grp_clks[i]);
 101                         rate_clk = gpu->grp_clks[i];
 102                 }
 103         }
 104
 105         if (rate_clk && gpu->fast_rate)
 106                 clk_set_rate(rate_clk, gpu->fast_rate);
 107
 108         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 109                 if (gpu->grp_clks[i])
 110                         clk_enable(gpu->grp_clks[i]);
 111
 112         return 0;
 113 }
 114
 115 static int disable_clk(struct msm_gpu *gpu)
 116 {
 117         struct clk *rate_clk = NULL;
 118         int i;
 119
 120         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
 121         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
 122                 if (gpu->grp_clks[i]) {
 123                         clk_disable(gpu->grp_clks[i]);
 124                         rate_clk = gpu->grp_clks[i];
 125                 }
 126         }
 127
 128         if (rate_clk && gpu->slow_rate)
 129                 clk_set_rate(rate_clk, gpu->slow_rate);
 130
 131         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 132                 if (gpu->grp_clks[i])
 133                         clk_unprepare(gpu->grp_clks[i]);
 134
 135         return 0;
 136 }
 137
 138 static int enable_axi(struct msm_gpu *gpu)
 139 {
 140         if (gpu->ebi1_clk)
 141                 clk_prepare_enable(gpu->ebi1_clk);
 142         if (gpu->bus_freq)
 143                 bs_set(gpu, gpu->bus_freq);
 144         return 0;
 145 }
 146
 147 static int disable_axi(struct msm_gpu *gpu)
 148 {
 149         if (gpu->ebi1_clk)
 150                 clk_disable_unprepare(gpu->ebi1_clk);
 151         if (gpu->bus_freq)
 152                 bs_set(gpu, 0);
 153         return 0;
 154 }
 155
 156 int msm_gpu_pm_resume(struct msm_gpu *gpu)
 157 {
 158         struct drm_device *dev = gpu->dev;
 159         int ret;
 160
 161         DBG("%s: active_cnt=%d", gpu->name, gpu->active_cnt);
 162
 163         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 164
 165         if (gpu->active_cnt++ > 0)
 166                 return 0;
 167
 168         if (WARN_ON(gpu->active_cnt <= 0))
 169                 return -EINVAL;
 170
 171         ret = enable_pwrrail(gpu);
 172         if (ret)
 173                 return ret;
 174
 175         ret = enable_clk(gpu);
 176         if (ret)
 177                 return ret;
 178
 179         ret = enable_axi(gpu);
 180         if (ret)
 181                 return ret;
 182
 183         return 0;
 184 }
 185
 186 int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 187 {
 188         struct drm_device *dev = gpu->dev;
 189         int ret;
 190
 191         DBG("%s: active_cnt=%d", gpu->name, gpu->active_cnt);
 192
 193         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 194
 195         if (--gpu->active_cnt > 0)
 196                 return 0;
 197
 198         if (WARN_ON(gpu->active_cnt < 0))
 199                 return -EINVAL;
 200
 201         ret = disable_axi(gpu);
 202         if (ret)
 203                 return ret;
 204
 205         ret = disable_clk(gpu);
 206         if (ret)
 207                 return ret;
 208
 209         ret = disable_pwrrail(gpu);
 210         if (ret)
 211                 return ret;
 212
 213         return 0;
 214 }
 215
 216 /*
 217  * Inactivity detection (for suspend):
 218  */
 219
 220 static void inactive_worker(struct work_struct *work)
 221 {
 222         struct msm_gpu *gpu = container_of(work, struct msm_gpu, inactive_work);
 223         struct drm_device *dev = gpu->dev;
 224
 225         if (gpu->inactive)
 226                 return;
 227
 228         DBG("%s: inactive!\n", gpu->name);
 229         mutex_lock(&dev->struct_mutex);
 230         if (!(msm_gpu_active(gpu) || gpu->inactive)) {
 231                 disable_axi(gpu);
 232                 disable_clk(gpu);
 233                 gpu->inactive = true;
 234         }
 235         mutex_unlock(&dev->struct_mutex);
 236 }
 237
 238 static void inactive_handler(unsigned long data)
 239 {
 240         struct msm_gpu *gpu = (struct msm_gpu *)data;
 241         struct msm_drm_private *priv = gpu->dev->dev_private;
 242
 243         queue_work(priv->wq, &gpu->inactive_work);
 244 }
 245
 246 /* cancel inactive timer and make sure we are awake: */
 247 static void inactive_cancel(struct msm_gpu *gpu)
 248 {
 249         DBG("%s", gpu->name);
 250         del_timer(&gpu->inactive_timer);
 251         if (gpu->inactive) {
 252                 enable_clk(gpu);
 253                 enable_axi(gpu);
 254                 gpu->inactive = false;
 255         }
 256 }
 257
 258 static void inactive_start(struct msm_gpu *gpu)
 259 {
 260         DBG("%s", gpu->name);
 261         mod_timer(&gpu->inactive_timer,
 262                         round_jiffies_up(jiffies + DRM_MSM_INACTIVE_JIFFIES));
 263 }
 264
 265 /*
 266  * Hangcheck detection for locked gpu:
 267  */
 268
 269 static void retire_submits(struct msm_gpu *gpu, uint32_t fence);
 270
 271 static void recover_worker(struct work_struct *work)
 272 {
 273         struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);
 274         struct drm_device *dev = gpu->dev;
 275
 276         dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name);
 277
 278         mutex_lock(&dev->struct_mutex);
 279         if (msm_gpu_active(gpu)) {
 280                 struct msm_gem_submit *submit;
 281                 uint32_t fence = gpu->funcs->last_fence(gpu);
 282
 283                 /* retire completed submits, plus the one that hung: */
 284                 retire_submits(gpu, fence + 1);
 285
 286                 inactive_cancel(gpu);
 287                 gpu->funcs->recover(gpu);
 288
 289                 /* replay the remaining submits after the one that hung: */
 290                 list_for_each_entry(submit, &gpu->submit_list, node) {
 291                         gpu->funcs->submit(gpu, submit, NULL);
 292                 }
 293         }
 294         mutex_unlock(&dev->struct_mutex);
 295
 296         msm_gpu_retire(gpu);
 297 }
 298
 299 static void hangcheck_timer_reset(struct msm_gpu *gpu)
 300 {
 301         DBG("%s", gpu->name);
 302         mod_timer(&gpu->hangcheck_timer,
 303                         round_jiffies_up(jiffies + DRM_MSM_HANGCHECK_JIFFIES));
 304 }
 305
 306 static void hangcheck_handler(unsigned long data)
 307 {
 308         struct msm_gpu *gpu = (struct msm_gpu *)data;
 309         struct drm_device *dev = gpu->dev;
 310         struct msm_drm_private *priv = dev->dev_private;
 311         uint32_t fence = gpu->funcs->last_fence(gpu);
 312
 313         if (fence != gpu->hangcheck_fence) {
 314                 /* some progress has been made.. ya! */
 315                 gpu->hangcheck_fence = fence;
 316         } else if (fence < gpu->submitted_fence) {
 317                 /* no progress and not done.. hung! */
 318                 gpu->hangcheck_fence = fence;
 319                 dev_err(dev->dev, "%s: hangcheck detected gpu lockup!\n",
 320                                 gpu->name);
 321                 dev_err(dev->dev, "%s:     completed fence: %u\n",
 322                                 gpu->name, fence);
 323                 dev_err(dev->dev, "%s:     submitted fence: %u\n",
 324                                 gpu->name, gpu->submitted_fence);
 325                 queue_work(priv->wq, &gpu->recover_work);
 326         }
 327
 328         /* if still more pending work, reset the hangcheck timer: */
 329         if (gpu->submitted_fence > gpu->hangcheck_fence)
 330                 hangcheck_timer_reset(gpu);
 331
 332         /* workaround for missing irq: */
 333         queue_work(priv->wq, &gpu->retire_work);
 334 }
 335
 336 /*
 337  * Performance Counters:
 338  */
 339
 340 /* called under perf_lock */
 341 static int update_hw_cntrs(struct msm_gpu *gpu, uint32_t ncntrs, uint32_t *cntrs)
 342 {
 343         uint32_t current_cntrs[ARRAY_SIZE(gpu->last_cntrs)];
 344         int i, n = min(ncntrs, gpu->num_perfcntrs);
 345
 346         /* read current values: */
 347         for (i = 0; i < gpu->num_perfcntrs; i++)
 348                 current_cntrs[i] = gpu_read(gpu, gpu->perfcntrs[i].sample_reg);
 349
 350         /* update cntrs: */
 351         for (i = 0; i < n; i++)
 352                 cntrs[i] = current_cntrs[i] - gpu->last_cntrs[i];
 353
 354         /* save current values: */
 355         for (i = 0; i < gpu->num_perfcntrs; i++)
 356                 gpu->last_cntrs[i] = current_cntrs[i];
 357
 358         return n;
 359 }
 360
 361 static void update_sw_cntrs(struct msm_gpu *gpu)
 362 {
 363         ktime_t time;
 364         uint32_t elapsed;
 365         unsigned long flags;
 366
 367         spin_lock_irqsave(&gpu->perf_lock, flags);
 368         if (!gpu->perfcntr_active)
 369                 goto out;
 370
 371         time = ktime_get();
 372         elapsed = ktime_to_us(ktime_sub(time, gpu->last_sample.time));
 373
 374         gpu->totaltime += elapsed;
 375         if (gpu->last_sample.active)
 376                 gpu->activetime += elapsed;
 377
 378         gpu->last_sample.active = msm_gpu_active(gpu);
 379         gpu->last_sample.time = time;
 380
 381 out:
 382         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 383 }
 384
 385 void msm_gpu_perfcntr_start(struct msm_gpu *gpu)
 386 {
 387         unsigned long flags;
 388
 389         spin_lock_irqsave(&gpu->perf_lock, flags);
 390         /* we could dynamically enable/disable perfcntr registers too.. */
 391         gpu->last_sample.active = msm_gpu_active(gpu);
 392         gpu->last_sample.time = ktime_get();
 393         gpu->activetime = gpu->totaltime = 0;
 394         gpu->perfcntr_active = true;
 395         update_hw_cntrs(gpu, 0, NULL);
 396         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 397 }
 398
 399 void msm_gpu_perfcntr_stop(struct msm_gpu *gpu)
 400 {
 401         gpu->perfcntr_active = false;
 402 }
 403
 404 /* returns -errno or # of cntrs sampled */
 405 int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
 406                 uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs)
 407 {
 408         unsigned long flags;
 409         int ret;
 410
 411         spin_lock_irqsave(&gpu->perf_lock, flags);
 412
 413         if (!gpu->perfcntr_active) {
 414                 ret = -EINVAL;
 415                 goto out;
 416         }
 417
 418         *activetime = gpu->activetime;
 419         *totaltime = gpu->totaltime;
 420
 421         gpu->activetime = gpu->totaltime = 0;
 422
 423         ret = update_hw_cntrs(gpu, ncntrs, cntrs);
 424
 425 out:
 426         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 427
 428         return ret;
 429 }
 430
 431 /*
 432  * Cmdstream submission/retirement:
 433  */
 434
 435 static void retire_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 436 {
 437         int i;
 438
 439         for (i = 0; i < submit->nr_bos; i++) {
 440                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 441                 /* move to inactive: */
 442                 msm_gem_move_to_inactive(&msm_obj->base);
 443                 msm_gem_put_iova(&msm_obj->base, gpu->id);
 444                 drm_gem_object_unreference(&msm_obj->base);
 445         }
 446
 447         list_del(&submit->node);
 448         kfree(submit);
 449 }
 450
 451 static void retire_submits(struct msm_gpu *gpu, uint32_t fence)
 452 {
 453         struct drm_device *dev = gpu->dev;
 454
 455         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 456
 457         while (!list_empty(&gpu->submit_list)) {
 458                 struct msm_gem_submit *submit;
 459
 460                 submit = list_first_entry(&gpu->submit_list,
 461                                 struct msm_gem_submit, node);
 462
 463                 if (submit->fence <= fence) {
 464                         retire_submit(gpu, submit);
 465                 } else {
 466                         break;
 467                 }
 468         }
 469 }
 470
 471 static void retire_worker(struct work_struct *work)
 472 {
 473         struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);
 474         struct drm_device *dev = gpu->dev;
 475         uint32_t fence = gpu->funcs->last_fence(gpu);
 476
 477         msm_update_fence(gpu->dev, fence);
 478
 479         mutex_lock(&dev->struct_mutex);
 480         retire_submits(gpu, fence);
 481         mutex_unlock(&dev->struct_mutex);
 482
 483         if (!msm_gpu_active(gpu))
 484                 inactive_start(gpu);
 485 }
 486
 487 /* call from irq handler to schedule work to retire bo's */
 488 void msm_gpu_retire(struct msm_gpu *gpu)
 489 {
 490         struct msm_drm_private *priv = gpu->dev->dev_private;
 491         queue_work(priv->wq, &gpu->retire_work);
 492         update_sw_cntrs(gpu);
 493 }
 494
 495 /* add bo's to gpu's ring, and kick gpu: */
 496 int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 497                 struct msm_file_private *ctx)
 498 {
 499         struct drm_device *dev = gpu->dev;
 500         struct msm_drm_private *priv = dev->dev_private;
 501         int i, ret;
 502
 503         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 504
 505         submit->fence = ++priv->next_fence;
 506
 507         gpu->submitted_fence = submit->fence;
 508
 509         inactive_cancel(gpu);
 510
 511         list_add_tail(&submit->node, &gpu->submit_list);
 512
 513         msm_rd_dump_submit(submit);
 514
 515         gpu->submitted_fence = submit->fence;
 516
 517         update_sw_cntrs(gpu);
 518
 519         for (i = 0; i < submit->nr_bos; i++) {
 520                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 521                 uint32_t iova;
 522
 523                 /* can't happen yet.. but when we add 2d support we'll have
 524                  * to deal w/ cross-ring synchronization:
 525                  */
 526                 WARN_ON(is_active(msm_obj) && (msm_obj->gpu != gpu));
 527
 528                 /* submit takes a reference to the bo and iova until retired: */
 529                 drm_gem_object_reference(&msm_obj->base);
 530                 msm_gem_get_iova_locked(&msm_obj->base,
 531                                 submit->gpu->id, &iova);
 532
 533                 if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
 534                         msm_gem_move_to_active(&msm_obj->base, gpu, false, submit->fence);
 535
 536                 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
 537                         msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence);
 538         }
 539
 540         ret = gpu->funcs->submit(gpu, submit, ctx);
 541         priv->lastctx = ctx;
 542
 543         hangcheck_timer_reset(gpu);
 544
 545         return ret;
 546 }
 547
 548 /*
 549  * Init/Cleanup:
 550  */
 551
 552 static irqreturn_t irq_handler(int irq, void *data)
 553 {
 554         struct msm_gpu *gpu = data;
 555         return gpu->funcs->irq(gpu);
 556 }
 557
 558 static const char *clk_names[] = {
 559                 "src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk",
 560                 "alt_mem_iface_clk",
 561 };
 562
 563 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 564                 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 565                 const char *name, const char *ioname, const char *irqname, int ringsz)
 566 {
 567         struct iommu_domain *iommu;
 568         int i, ret;
 569
 570         if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs)))
 571                 gpu->num_perfcntrs = ARRAY_SIZE(gpu->last_cntrs);
 572
 573         gpu->dev = drm;
 574         gpu->funcs = funcs;
 575         gpu->name = name;
 576         gpu->inactive = true;
 577
 578         INIT_LIST_HEAD(&gpu->active_list);
 579         INIT_WORK(&gpu->retire_work, retire_worker);
 580         INIT_WORK(&gpu->inactive_work, inactive_worker);
 581         INIT_WORK(&gpu->recover_work, recover_worker);
 582
 583         INIT_LIST_HEAD(&gpu->submit_list);
 584
 585         setup_timer(&gpu->inactive_timer, inactive_handler,
 586                         (unsigned long)gpu);
 587         setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
 588                         (unsigned long)gpu);
 589
 590         spin_lock_init(&gpu->perf_lock);
 591
 592         BUG_ON(ARRAY_SIZE(clk_names) != ARRAY_SIZE(gpu->grp_clks));
 593
 594         /* Map registers: */
 595         gpu->mmio = msm_ioremap(pdev, ioname, name);
 596         if (IS_ERR(gpu->mmio)) {
 597                 ret = PTR_ERR(gpu->mmio);
 598                 goto fail;
 599         }
 600
 601         /* Get Interrupt: */
 602         gpu->irq = platform_get_irq_byname(pdev, irqname);
 603         if (gpu->irq < 0) {
 604                 ret = gpu->irq;
 605                 dev_err(drm->dev, "failed to get irq: %d\n", ret);
 606                 goto fail;
 607         }
 608
 609         ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler,
 610                         IRQF_TRIGGER_HIGH, gpu->name, gpu);
 611         if (ret) {
 612                 dev_err(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);
 613                 goto fail;
 614         }
 615
 616         /* Acquire clocks: */
 617         for (i = 0; i < ARRAY_SIZE(clk_names); i++) {
 618                 gpu->grp_clks[i] = devm_clk_get(&pdev->dev, clk_names[i]);
 619                 DBG("grp_clks[%s]: %p", clk_names[i], gpu->grp_clks[i]);
 620                 if (IS_ERR(gpu->grp_clks[i]))
 621                         gpu->grp_clks[i] = NULL;
 622         }
 623
 624         gpu->ebi1_clk = devm_clk_get(&pdev->dev, "bus_clk");
 625         DBG("ebi1_clk: %p", gpu->ebi1_clk);
 626         if (IS_ERR(gpu->ebi1_clk))
 627                 gpu->ebi1_clk = NULL;
 628
 629         /* Acquire regulators: */
 630         gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd");
 631         DBG("gpu_reg: %p", gpu->gpu_reg);
 632         if (IS_ERR(gpu->gpu_reg))
 633                 gpu->gpu_reg = NULL;
 634
 635         gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx");
 636         DBG("gpu_cx: %p", gpu->gpu_cx);
 637         if (IS_ERR(gpu->gpu_cx))
 638                 gpu->gpu_cx = NULL;
 639
 640         /* Setup IOMMU.. eventually we will (I think) do this once per context
 641          * and have separate page tables per context.  For now, to keep things
 642          * simple and to get something working, just use a single address space:
 643          */
 644         iommu = iommu_domain_alloc(&platform_bus_type);
 645         if (iommu) {
 646                 dev_info(drm->dev, "%s: using IOMMU\n", name);
 647                 gpu->mmu = msm_iommu_new(&pdev->dev, iommu);
 648                 if (IS_ERR(gpu->mmu)) {
 649                         ret = PTR_ERR(gpu->mmu);
 650                         dev_err(drm->dev, "failed to init iommu: %d\n", ret);
 651                         gpu->mmu = NULL;
 652                         iommu_domain_free(iommu);
 653                         goto fail;
 654                 }
 655
 656         } else {
 657                 dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
 658         }
 659         gpu->id = msm_register_mmu(drm, gpu->mmu);
 660
 661
 662         /* Create ringbuffer: */
 663         mutex_lock(&drm->struct_mutex);
 664         gpu->rb = msm_ringbuffer_new(gpu, ringsz);
 665         mutex_unlock(&drm->struct_mutex);
 666         if (IS_ERR(gpu->rb)) {
 667                 ret = PTR_ERR(gpu->rb);
 668                 gpu->rb = NULL;
 669                 dev_err(drm->dev, "could not create ringbuffer: %d\n", ret);
 670                 goto fail;
 671         }
 672
 673         bs_init(gpu);
 674
 675         return 0;
 676
 677 fail:
 678         return ret;
 679 }
 680
 681 void msm_gpu_cleanup(struct msm_gpu *gpu)
 682 {
 683         DBG("%s", gpu->name);
 684
 685         WARN_ON(!list_empty(&gpu->active_list));
 686
 687         bs_fini(gpu);
 688
 689         if (gpu->rb) {
 690                 if (gpu->rb_iova)
 691                         msm_gem_put_iova(gpu->rb->bo, gpu->id);
 692                 msm_ringbuffer_destroy(gpu->rb);
 693         }
 694
 695         if (gpu->mmu)
 696                 gpu->mmu->funcs->destroy(gpu->mmu);
 697 }