drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/fence-array.h>
  29 #include <drm/drmP.h>
  30 #include <drm/amdgpu_drm.h>
  31 #include "amdgpu.h"
  32 #include "amdgpu_trace.h"
  33
  34 /*
  35  * GPUVM
  36  * GPUVM is similar to the legacy gart on older asics, however
  37  * rather than there being a single global gart table
  38  * for the entire GPU, there are multiple VM page tables active
  39  * at any given time.  The VM page tables can contain a mix
  40  * vram pages and system memory pages and system memory pages
  41  * can be mapped as snooped (cached system pages) or unsnooped
  42  * (uncached system pages).
  43  * Each VM has an ID associated with it and there is a page table
  44  * associated with each VMID.  When execting a command buffer,
  45  * the kernel tells the the ring what VMID to use for that command
  46  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  47  * The userspace drivers maintain their own address space and the kernel
  48  * sets up their pages tables accordingly when they submit their
  49  * command buffers and a VMID is assigned.
  50  * Cayman/Trinity support up to 8 active VMs at any given time;
  51  * SI supports 16.
  52  */
  53
  54 /* Local structure. Encapsulate some VM table update parameters to reduce
  55  * the number of function parameters
  56  */
  57 struct amdgpu_pte_update_params {
  58         /* amdgpu device we do this update for */
  59         struct amdgpu_device *adev;
  60         /* address where to copy page table entries from */
  61         uint64_t src;
  62         /* indirect buffer to fill with commands */
  63         struct amdgpu_ib *ib;
  64         /* Function which actually does the update */
  65         void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  66                      uint64_t addr, unsigned count, uint32_t incr,
  67                      uint32_t flags);
  68         /* indicate update pt or its shadow */
  69         bool shadow;
  70 };
  71
  72 /**
  73  * amdgpu_vm_num_pde - return the number of page directory entries
  74  *
  75  * @adev: amdgpu_device pointer
  76  *
  77  * Calculate the number of page directory entries.
  78  */
  79 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
  80 {
  81         return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
  82 }
  83
  84 /**
  85  * amdgpu_vm_directory_size - returns the size of the page directory in bytes
  86  *
  87  * @adev: amdgpu_device pointer
  88  *
  89  * Calculate the size of the page directory in bytes.
  90  */
  91 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
  92 {
  93         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
  94 }
  95
  96 /**
  97  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  98  *
  99  * @vm: vm providing the BOs
 100  * @validated: head of validation list
 101  * @entry: entry to add
 102  *
 103  * Add the page directory to the list of BOs to
 104  * validate for command submission.
 105  */
 106 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 107                          struct list_head *validated,
 108                          struct amdgpu_bo_list_entry *entry)
 109 {
 110         entry->robj = vm->page_directory;
 111         entry->priority = 0;
 112         entry->tv.bo = &vm->page_directory->tbo;
 113         entry->tv.shared = true;
 114         entry->user_pages = NULL;
 115         list_add(&entry->tv.head, validated);
 116 }
 117
 118 /**
 119  * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
 120  *
 121  * @adev: amdgpu device pointer
 122  * @vm: vm providing the BOs
 123  * @duplicates: head of duplicates list
 124  *
 125  * Add the page directory to the BO duplicates list
 126  * for command submission.
 127  */
 128 void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 129                           struct list_head *duplicates)
 130 {
 131         uint64_t num_evictions;
 132         unsigned i;
 133
 134         /* We only need to validate the page tables
 135          * if they aren't already valid.
 136          */
 137         num_evictions = atomic64_read(&adev->num_evictions);
 138         if (num_evictions == vm->last_eviction_counter)
 139                 return;
 140
 141         /* add the vm page table to the list */
 142         for (i = 0; i <= vm->max_pde_used; ++i) {
 143                 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
 144
 145                 if (!entry->robj)
 146                         continue;
 147
 148                 list_add(&entry->tv.head, duplicates);
 149         }
 150
 151 }
 152
 153 /**
 154  * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 155  *
 156  * @adev: amdgpu device instance
 157  * @vm: vm providing the BOs
 158  *
 159  * Move the PT BOs to the tail of the LRU.
 160  */
 161 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
 162                                   struct amdgpu_vm *vm)
 163 {
 164         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 165         unsigned i;
 166
 167         spin_lock(&glob->lru_lock);
 168         for (i = 0; i <= vm->max_pde_used; ++i) {
 169                 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
 170
 171                 if (!entry->robj)
 172                         continue;
 173
 174                 ttm_bo_move_to_lru_tail(&entry->robj->tbo);
 175         }
 176         spin_unlock(&glob->lru_lock);
 177 }
 178
 179 static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
 180                               struct amdgpu_vm_id *id)
 181 {
 182         return id->current_gpu_reset_count !=
 183                 atomic_read(&adev->gpu_reset_counter) ? true : false;
 184 }
 185
 186 /**
 187  * amdgpu_vm_grab_id - allocate the next free VMID
 188  *
 189  * @vm: vm to allocate id for
 190  * @ring: ring we want to submit job to
 191  * @sync: sync object where we add dependencies
 192  * @fence: fence protecting ID from reuse
 193  *
 194  * Allocate an id for the vm, adding fences to the sync obj as necessary.
 195  */
 196 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 197                       struct amdgpu_sync *sync, struct fence *fence,
 198                       struct amdgpu_job *job)
 199 {
 200         struct amdgpu_device *adev = ring->adev;
 201         uint64_t fence_context = adev->fence_context + ring->idx;
 202         struct fence *updates = sync->last_vm_update;
 203         struct amdgpu_vm_id *id, *idle;
 204         struct fence **fences;
 205         unsigned i;
 206         int r = 0;
 207
 208         fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
 209                                GFP_KERNEL);
 210         if (!fences)
 211                 return -ENOMEM;
 212
 213         mutex_lock(&adev->vm_manager.lock);
 214
 215         /* Check if we have an idle VMID */
 216         i = 0;
 217         list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
 218                 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
 219                 if (!fences[i])
 220                         break;
 221                 ++i;
 222         }
 223
 224         /* If we can't find a idle VMID to use, wait till one becomes available */
 225         if (&idle->list == &adev->vm_manager.ids_lru) {
 226                 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
 227                 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
 228                 struct fence_array *array;
 229                 unsigned j;
 230
 231                 for (j = 0; j < i; ++j)
 232                         fence_get(fences[j]);
 233
 234                 array = fence_array_create(i, fences, fence_context,
 235                                            seqno, true);
 236                 if (!array) {
 237                         for (j = 0; j < i; ++j)
 238                                 fence_put(fences[j]);
 239                         kfree(fences);
 240                         r = -ENOMEM;
 241                         goto error;
 242                 }
 243
 244
 245                 r = amdgpu_sync_fence(ring->adev, sync, &array->base);
 246                 fence_put(&array->base);
 247                 if (r)
 248                         goto error;
 249
 250                 mutex_unlock(&adev->vm_manager.lock);
 251                 return 0;
 252
 253         }
 254         kfree(fences);
 255
 256         job->vm_needs_flush = true;
 257         /* Check if we can use a VMID already assigned to this VM */
 258         i = ring->idx;
 259         do {
 260                 struct fence *flushed;
 261
 262                 id = vm->ids[i++];
 263                 if (i == AMDGPU_MAX_RINGS)
 264                         i = 0;
 265
 266                 /* Check all the prerequisites to using this VMID */
 267                 if (!id)
 268                         continue;
 269                 if (amdgpu_vm_is_gpu_reset(adev, id))
 270                         continue;
 271
 272                 if (atomic64_read(&id->owner) != vm->client_id)
 273                         continue;
 274
 275                 if (job->vm_pd_addr != id->pd_gpu_addr)
 276                         continue;
 277
 278                 if (!id->last_flush)
 279                         continue;
 280
 281                 if (id->last_flush->context != fence_context &&
 282                     !fence_is_signaled(id->last_flush))
 283                         continue;
 284
 285                 flushed  = id->flushed_updates;
 286                 if (updates &&
 287                     (!flushed || fence_is_later(updates, flushed)))
 288                         continue;
 289
 290                 /* Good we can use this VMID. Remember this submission as
 291                  * user of the VMID.
 292                  */
 293                 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 294                 if (r)
 295                         goto error;
 296
 297                 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 298                 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 299                 vm->ids[ring->idx] = id;
 300
 301                 job->vm_id = id - adev->vm_manager.ids;
 302                 job->vm_needs_flush = false;
 303                 trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 304
 305                 mutex_unlock(&adev->vm_manager.lock);
 306                 return 0;
 307
 308         } while (i != ring->idx);
 309
 310         /* Still no ID to use? Then use the idle one found earlier */
 311         id = idle;
 312
 313         /* Remember this submission as user of the VMID */
 314         r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 315         if (r)
 316                 goto error;
 317
 318         fence_put(id->first);
 319         id->first = fence_get(fence);
 320
 321         fence_put(id->last_flush);
 322         id->last_flush = NULL;
 323
 324         fence_put(id->flushed_updates);
 325         id->flushed_updates = fence_get(updates);
 326
 327         id->pd_gpu_addr = job->vm_pd_addr;
 328         id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 329         list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 330         atomic64_set(&id->owner, vm->client_id);
 331         vm->ids[ring->idx] = id;
 332
 333         job->vm_id = id - adev->vm_manager.ids;
 334         trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 335
 336 error:
 337         mutex_unlock(&adev->vm_manager.lock);
 338         return r;
 339 }
 340
 341 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
 342 {
 343         struct amdgpu_device *adev = ring->adev;
 344         const struct amdgpu_ip_block_version *ip_block;
 345
 346         if (ring->type != AMDGPU_RING_TYPE_COMPUTE)
 347                 /* only compute rings */
 348                 return false;
 349
 350         ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 351         if (!ip_block)
 352                 return false;
 353
 354         if (ip_block->major <= 7) {
 355                 /* gfx7 has no workaround */
 356                 return true;
 357         } else if (ip_block->major == 8) {
 358                 if (adev->gfx.mec_fw_version >= 673)
 359                         /* gfx8 is fixed in MEC firmware 673 */
 360                         return false;
 361                 else
 362                         return true;
 363         }
 364         return false;
 365 }
 366
 367 /**
 368  * amdgpu_vm_flush - hardware flush the vm
 369  *
 370  * @ring: ring to use for flush
 371  * @vm_id: vmid number to use
 372  * @pd_addr: address of the page directory
 373  *
 374  * Emit a VM flush when it is necessary.
 375  */
 376 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 377 {
 378         struct amdgpu_device *adev = ring->adev;
 379         struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id];
 380         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 381                 id->gds_base != job->gds_base ||
 382                 id->gds_size != job->gds_size ||
 383                 id->gws_base != job->gws_base ||
 384                 id->gws_size != job->gws_size ||
 385                 id->oa_base != job->oa_base ||
 386                 id->oa_size != job->oa_size);
 387         int r;
 388
 389         if (ring->funcs->emit_pipeline_sync && (
 390             job->vm_needs_flush || gds_switch_needed ||
 391             amdgpu_vm_ring_has_compute_vm_bug(ring)))
 392                 amdgpu_ring_emit_pipeline_sync(ring);
 393
 394         if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
 395             amdgpu_vm_is_gpu_reset(adev, id))) {
 396                 struct fence *fence;
 397
 398                 trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
 399                 amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
 400
 401                 r = amdgpu_fence_emit(ring, &fence);
 402                 if (r)
 403                         return r;
 404
 405                 mutex_lock(&adev->vm_manager.lock);
 406                 fence_put(id->last_flush);
 407                 id->last_flush = fence;
 408                 mutex_unlock(&adev->vm_manager.lock);
 409         }
 410
 411         if (gds_switch_needed) {
 412                 id->gds_base = job->gds_base;
 413                 id->gds_size = job->gds_size;
 414                 id->gws_base = job->gws_base;
 415                 id->gws_size = job->gws_size;
 416                 id->oa_base = job->oa_base;
 417                 id->oa_size = job->oa_size;
 418                 amdgpu_ring_emit_gds_switch(ring, job->vm_id,
 419                                             job->gds_base, job->gds_size,
 420                                             job->gws_base, job->gws_size,
 421                                             job->oa_base, job->oa_size);
 422         }
 423
 424         return 0;
 425 }
 426
 427 /**
 428  * amdgpu_vm_reset_id - reset VMID to zero
 429  *
 430  * @adev: amdgpu device structure
 431  * @vm_id: vmid number to use
 432  *
 433  * Reset saved GDW, GWS and OA to force switch on next flush.
 434  */
 435 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
 436 {
 437         struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
 438
 439         id->gds_base = 0;
 440         id->gds_size = 0;
 441         id->gws_base = 0;
 442         id->gws_size = 0;
 443         id->oa_base = 0;
 444         id->oa_size = 0;
 445 }
 446
 447 /**
 448  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 449  *
 450  * @vm: requested vm
 451  * @bo: requested buffer object
 452  *
 453  * Find @bo inside the requested vm.
 454  * Search inside the @bos vm list for the requested vm
 455  * Returns the found bo_va or NULL if none is found
 456  *
 457  * Object has to be reserved!
 458  */
 459 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 460                                        struct amdgpu_bo *bo)
 461 {
 462         struct amdgpu_bo_va *bo_va;
 463
 464         list_for_each_entry(bo_va, &bo->va, bo_list) {
 465                 if (bo_va->vm == vm) {
 466                         return bo_va;
 467                 }
 468         }
 469         return NULL;
 470 }
 471
 472 /**
 473  * amdgpu_vm_do_set_ptes - helper to call the right asic function
 474  *
 475  * @params: see amdgpu_pte_update_params definition
 476  * @pe: addr of the page entry
 477  * @addr: dst addr to write into pe
 478  * @count: number of page entries to update
 479  * @incr: increase next addr by incr bytes
 480  * @flags: hw access flags
 481  *
 482  * Traces the parameters and calls the right asic functions
 483  * to setup the page table using the DMA.
 484  */
 485 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
 486                                   uint64_t pe, uint64_t addr,
 487                                   unsigned count, uint32_t incr,
 488                                   uint32_t flags)
 489 {
 490         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 491
 492         if (count < 3) {
 493                 amdgpu_vm_write_pte(params->adev, params->ib, pe,
 494                                     addr | flags, count, incr);
 495
 496         } else {
 497                 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
 498                                       count, incr, flags);
 499         }
 500 }
 501
 502 /**
 503  * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 504  *
 505  * @params: see amdgpu_pte_update_params definition
 506  * @pe: addr of the page entry
 507  * @addr: dst addr to write into pe
 508  * @count: number of page entries to update
 509  * @incr: increase next addr by incr bytes
 510  * @flags: hw access flags
 511  *
 512  * Traces the parameters and calls the DMA function to copy the PTEs.
 513  */
 514 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 515                                    uint64_t pe, uint64_t addr,
 516                                    unsigned count, uint32_t incr,
 517                                    uint32_t flags)
 518 {
 519         uint64_t src = (params->src + (addr >> 12) * 8);
 520
 521
 522         trace_amdgpu_vm_copy_ptes(pe, src, count);
 523
 524         amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 525 }
 526
 527 /**
 528  * amdgpu_vm_clear_bo - initially clear the page dir/table
 529  *
 530  * @adev: amdgpu_device pointer
 531  * @bo: bo to clear
 532  *
 533  * need to reserve bo first before calling it.
 534  */
 535 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 536                               struct amdgpu_vm *vm,
 537                               struct amdgpu_bo *bo)
 538 {
 539         struct amdgpu_ring *ring;
 540         struct fence *fence = NULL;
 541         struct amdgpu_job *job;
 542         struct amdgpu_pte_update_params params;
 543         unsigned entries;
 544         uint64_t addr;
 545         int r;
 546
 547         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 548
 549         r = reservation_object_reserve_shared(bo->tbo.resv);
 550         if (r)
 551                 return r;
 552
 553         r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
 554         if (r)
 555                 goto error;
 556
 557         r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
 558         if (r)
 559                 goto error;
 560
 561         addr = amdgpu_bo_gpu_offset(bo);
 562         entries = amdgpu_bo_size(bo) / 8;
 563
 564         r = amdgpu_job_alloc_with_ib(adev, 64, &job);
 565         if (r)
 566                 goto error;
 567
 568         memset(&params, 0, sizeof(params));
 569         params.adev = adev;
 570         params.ib = &job->ibs[0];
 571         amdgpu_vm_do_set_ptes(&params, addr, 0, entries, 0, 0);
 572         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 573
 574         WARN_ON(job->ibs[0].length_dw > 64);
 575         r = amdgpu_job_submit(job, ring, &vm->entity,
 576                               AMDGPU_FENCE_OWNER_VM, &fence);
 577         if (r)
 578                 goto error_free;
 579
 580         amdgpu_bo_fence(bo, fence, true);
 581         fence_put(fence);
 582         return 0;
 583
 584 error_free:
 585         amdgpu_job_free(job);
 586
 587 error:
 588         return r;
 589 }
 590
 591 /**
 592  * amdgpu_vm_map_gart - Resolve gart mapping of addr
 593  *
 594  * @pages_addr: optional DMA address to use for lookup
 595  * @addr: the unmapped addr
 596  *
 597  * Look up the physical address of the page that the pte resolves
 598  * to and return the pointer for the page table entry.
 599  */
 600 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 601 {
 602         uint64_t result;
 603
 604         /* page table offset */
 605         result = pages_addr[addr >> PAGE_SHIFT];
 606
 607         /* in case cpu page size != gpu page size*/
 608         result |= addr & (~PAGE_MASK);
 609
 610         result &= 0xFFFFFFFFFFFFF000ULL;
 611
 612         return result;
 613 }
 614
 615 static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
 616                                          struct amdgpu_vm *vm,
 617                                          bool shadow)
 618 {
 619         struct amdgpu_ring *ring;
 620         struct amdgpu_bo *pd = shadow ? vm->page_directory->shadow :
 621                 vm->page_directory;
 622         uint64_t pd_addr;
 623         uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
 624         uint64_t last_pde = ~0, last_pt = ~0;
 625         unsigned count = 0, pt_idx, ndw;
 626         struct amdgpu_job *job;
 627         struct amdgpu_pte_update_params params;
 628         struct fence *fence = NULL;
 629
 630         int r;
 631
 632         if (!pd)
 633                 return 0;
 634
 635         r = amdgpu_ttm_bind(&pd->tbo, &pd->tbo.mem);
 636         if (r)
 637                 return r;
 638
 639         pd_addr = amdgpu_bo_gpu_offset(pd);
 640         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 641
 642         /* padding, etc. */
 643         ndw = 64;
 644
 645         /* assume the worst case */
 646         ndw += vm->max_pde_used * 6;
 647
 648         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 649         if (r)
 650                 return r;
 651
 652         memset(&params, 0, sizeof(params));
 653         params.adev = adev;
 654         params.ib = &job->ibs[0];
 655
 656         /* walk over the address space and update the page directory */
 657         for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
 658                 struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
 659                 uint64_t pde, pt;
 660
 661                 if (bo == NULL)
 662                         continue;
 663
 664                 if (bo->shadow) {
 665                         struct amdgpu_bo *shadow = bo->shadow;
 666
 667                         r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
 668                         if (r)
 669                                 return r;
 670                 }
 671
 672                 pt = amdgpu_bo_gpu_offset(bo);
 673                 if (!shadow) {
 674                         if (vm->page_tables[pt_idx].addr == pt)
 675                                 continue;
 676                         vm->page_tables[pt_idx].addr = pt;
 677                 } else {
 678                         if (vm->page_tables[pt_idx].shadow_addr == pt)
 679                                 continue;
 680                         vm->page_tables[pt_idx].shadow_addr = pt;
 681                 }
 682
 683                 pde = pd_addr + pt_idx * 8;
 684                 if (((last_pde + 8 * count) != pde) ||
 685                     ((last_pt + incr * count) != pt) ||
 686                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
 687
 688                         if (count) {
 689                                 amdgpu_vm_do_set_ptes(&params, last_pde,
 690                                                       last_pt, count, incr,
 691                                                       AMDGPU_PTE_VALID);
 692                         }
 693
 694                         count = 1;
 695                         last_pde = pde;
 696                         last_pt = pt;
 697                 } else {
 698                         ++count;
 699                 }
 700         }
 701
 702         if (count)
 703                 amdgpu_vm_do_set_ptes(&params, last_pde, last_pt,
 704                                       count, incr, AMDGPU_PTE_VALID);
 705
 706         if (params.ib->length_dw != 0) {
 707                 amdgpu_ring_pad_ib(ring, params.ib);
 708                 amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
 709                                  AMDGPU_FENCE_OWNER_VM);
 710                 WARN_ON(params.ib->length_dw > ndw);
 711                 r = amdgpu_job_submit(job, ring, &vm->entity,
 712                                       AMDGPU_FENCE_OWNER_VM, &fence);
 713                 if (r)
 714                         goto error_free;
 715
 716                 amdgpu_bo_fence(pd, fence, true);
 717                 fence_put(vm->page_directory_fence);
 718                 vm->page_directory_fence = fence_get(fence);
 719                 fence_put(fence);
 720
 721         } else {
 722                 amdgpu_job_free(job);
 723         }
 724
 725         return 0;
 726
 727 error_free:
 728         amdgpu_job_free(job);
 729         return r;
 730 }
 731
 732 /*
 733  * amdgpu_vm_update_pdes - make sure that page directory is valid
 734  *
 735  * @adev: amdgpu_device pointer
 736  * @vm: requested vm
 737  * @start: start of GPU address range
 738  * @end: end of GPU address range
 739  *
 740  * Allocates new page tables if necessary
 741  * and updates the page directory.
 742  * Returns 0 for success, error for failure.
 743  */
 744 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 745                                    struct amdgpu_vm *vm)
 746 {
 747         int r;
 748
 749         r = amdgpu_vm_update_pd_or_shadow(adev, vm, true);
 750         if (r)
 751                 return r;
 752         return amdgpu_vm_update_pd_or_shadow(adev, vm, false);
 753 }
 754
 755 /**
 756  * amdgpu_vm_update_ptes - make sure that page tables are valid
 757  *
 758  * @params: see amdgpu_pte_update_params definition
 759  * @vm: requested vm
 760  * @start: start of GPU address range
 761  * @end: end of GPU address range
 762  * @dst: destination address to map to, the next dst inside the function
 763  * @flags: mapping flags
 764  *
 765  * Update the page tables in the range @start - @end.
 766  */
 767 static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 768                                   struct amdgpu_vm *vm,
 769                                   uint64_t start, uint64_t end,
 770                                   uint64_t dst, uint32_t flags)
 771 {
 772         const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
 773
 774         uint64_t cur_pe_start, cur_nptes, cur_dst;
 775         uint64_t addr; /* next GPU address to be updated */
 776         uint64_t pt_idx;
 777         struct amdgpu_bo *pt;
 778         unsigned nptes; /* next number of ptes to be updated */
 779         uint64_t next_pe_start;
 780
 781         /* initialize the variables */
 782         addr = start;
 783         pt_idx = addr >> amdgpu_vm_block_size;
 784         pt = vm->page_tables[pt_idx].entry.robj;
 785         if (params->shadow) {
 786                 if (!pt->shadow)
 787                         return;
 788                 pt = vm->page_tables[pt_idx].entry.robj->shadow;
 789         }
 790         if ((addr & ~mask) == (end & ~mask))
 791                 nptes = end - addr;
 792         else
 793                 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 794
 795         cur_pe_start = amdgpu_bo_gpu_offset(pt);
 796         cur_pe_start += (addr & mask) * 8;
 797         cur_nptes = nptes;
 798         cur_dst = dst;
 799
 800         /* for next ptb*/
 801         addr += nptes;
 802         dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 803
 804         /* walk over the address space and update the page tables */
 805         while (addr < end) {
 806                 pt_idx = addr >> amdgpu_vm_block_size;
 807                 pt = vm->page_tables[pt_idx].entry.robj;
 808                 if (params->shadow) {
 809                         if (!pt->shadow)
 810                                 return;
 811                         pt = vm->page_tables[pt_idx].entry.robj->shadow;
 812                 }
 813
 814                 if ((addr & ~mask) == (end & ~mask))
 815                         nptes = end - addr;
 816                 else
 817                         nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 818
 819                 next_pe_start = amdgpu_bo_gpu_offset(pt);
 820                 next_pe_start += (addr & mask) * 8;
 821
 822                 if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
 823                     ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
 824                         /* The next ptb is consecutive to current ptb.
 825                          * Don't call the update function now.
 826                          * Will update two ptbs together in future.
 827                         */
 828                         cur_nptes += nptes;
 829                 } else {
 830                         params->func(params, cur_pe_start, cur_dst, cur_nptes,
 831                                      AMDGPU_GPU_PAGE_SIZE, flags);
 832
 833                         cur_pe_start = next_pe_start;
 834                         cur_nptes = nptes;
 835                         cur_dst = dst;
 836                 }
 837
 838                 /* for next ptb*/
 839                 addr += nptes;
 840                 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 841         }
 842
 843         params->func(params, cur_pe_start, cur_dst, cur_nptes,
 844                      AMDGPU_GPU_PAGE_SIZE, flags);
 845 }
 846
 847 /*
 848  * amdgpu_vm_frag_ptes - add fragment information to PTEs
 849  *
 850  * @params: see amdgpu_pte_update_params definition
 851  * @vm: requested vm
 852  * @start: first PTE to handle
 853  * @end: last PTE to handle
 854  * @dst: addr those PTEs should point to
 855  * @flags: hw mapping flags
 856  */
 857 static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
 858                                 struct amdgpu_vm *vm,
 859                                 uint64_t start, uint64_t end,
 860                                 uint64_t dst, uint32_t flags)
 861 {
 862         /**
 863          * The MC L1 TLB supports variable sized pages, based on a fragment
 864          * field in the PTE. When this field is set to a non-zero value, page
 865          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
 866          * flags are considered valid for all PTEs within the fragment range
 867          * and corresponding mappings are assumed to be physically contiguous.
 868          *
 869          * The L1 TLB can store a single PTE for the whole fragment,
 870          * significantly increasing the space available for translation
 871          * caching. This leads to large improvements in throughput when the
 872          * TLB is under pressure.
 873          *
 874          * The L2 TLB distributes small and large fragments into two
 875          * asymmetric partitions. The large fragment cache is significantly
 876          * larger. Thus, we try to use large fragments wherever possible.
 877          * Userspace can support this by aligning virtual base address and
 878          * allocation size to the fragment size.
 879          */
 880
 881         const uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
 882
 883         uint64_t frag_start = ALIGN(start, frag_align);
 884         uint64_t frag_end = end & ~(frag_align - 1);
 885
 886         uint32_t frag;
 887
 888         /* system pages are non continuously */
 889         if (params->src || !(flags & AMDGPU_PTE_VALID) ||
 890             (frag_start >= frag_end)) {
 891
 892                 amdgpu_vm_update_ptes(params, vm, start, end, dst, flags);
 893                 return;
 894         }
 895
 896         /* use more than 64KB fragment size if possible */
 897         frag = lower_32_bits(frag_start | frag_end);
 898         frag = likely(frag) ? __ffs(frag) : 31;
 899
 900         /* handle the 4K area at the beginning */
 901         if (start != frag_start) {
 902                 amdgpu_vm_update_ptes(params, vm, start, frag_start,
 903                                       dst, flags);
 904                 dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
 905         }
 906
 907         /* handle the area in the middle */
 908         amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
 909                               flags | AMDGPU_PTE_FRAG(frag));
 910
 911         /* handle the 4K area at the end */
 912         if (frag_end != end) {
 913                 dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
 914                 amdgpu_vm_update_ptes(params, vm, frag_end, end, dst, flags);
 915         }
 916 }
 917
 918 /**
 919  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 920  *
 921  * @adev: amdgpu_device pointer
 922  * @exclusive: fence we need to sync to
 923  * @src: address where to copy page table entries from
 924  * @pages_addr: DMA addresses to use for mapping
 925  * @vm: requested vm
 926  * @start: start of mapped range
 927  * @last: last mapped entry
 928  * @flags: flags for the entries
 929  * @addr: addr to set the area to
 930  * @fence: optional resulting fence
 931  *
 932  * Fill in the page table entries between @start and @last.
 933  * Returns 0 for success, -EINVAL for failure.
 934  */
 935 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 936                                        struct fence *exclusive,
 937                                        uint64_t src,
 938                                        dma_addr_t *pages_addr,
 939                                        struct amdgpu_vm *vm,
 940                                        uint64_t start, uint64_t last,
 941                                        uint32_t flags, uint64_t addr,
 942                                        struct fence **fence)
 943 {
 944         struct amdgpu_ring *ring;
 945         void *owner = AMDGPU_FENCE_OWNER_VM;
 946         unsigned nptes, ncmds, ndw;
 947         struct amdgpu_job *job;
 948         struct amdgpu_pte_update_params params;
 949         struct fence *f = NULL;
 950         int r;
 951
 952         memset(&params, 0, sizeof(params));
 953         params.adev = adev;
 954         params.src = src;
 955
 956         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 957
 958         memset(&params, 0, sizeof(params));
 959         params.adev = adev;
 960         params.src = src;
 961
 962         /* sync to everything on unmapping */
 963         if (!(flags & AMDGPU_PTE_VALID))
 964                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
 965
 966         nptes = last - start + 1;
 967
 968         /*
 969          * reserve space for one command every (1 << BLOCK_SIZE)
 970          *  entries or 2k dwords (whatever is smaller)
 971          */
 972         ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
 973
 974         /* padding, etc. */
 975         ndw = 64;
 976
 977         if (src) {
 978                 /* only copy commands needed */
 979                 ndw += ncmds * 7;
 980
 981                 params.func = amdgpu_vm_do_copy_ptes;
 982
 983         } else if (pages_addr) {
 984                 /* copy commands needed */
 985                 ndw += ncmds * 7;
 986
 987                 /* and also PTEs */
 988                 ndw += nptes * 2;
 989
 990                 params.func = amdgpu_vm_do_copy_ptes;
 991
 992         } else {
 993                 /* set page commands needed */
 994                 ndw += ncmds * 10;
 995
 996                 /* two extra commands for begin/end of fragment */
 997                 ndw += 2 * 10;
 998
 999                 params.func = amdgpu_vm_do_set_ptes;
1000         }
1001
1002         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1003         if (r)
1004                 return r;
1005
1006         params.ib = &job->ibs[0];
1007
1008         if (!src && pages_addr) {
1009                 uint64_t *pte;
1010                 unsigned i;
1011
1012                 /* Put the PTEs at the end of the IB. */
1013                 i = ndw - nptes * 2;
1014                 pte= (uint64_t *)&(job->ibs->ptr[i]);
1015                 params.src = job->ibs->gpu_addr + i * 4;
1016
1017                 for (i = 0; i < nptes; ++i) {
1018                         pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1019                                                     AMDGPU_GPU_PAGE_SIZE);
1020                         pte[i] |= flags;
1021                 }
1022                 addr = 0;
1023         }
1024
1025         r = amdgpu_sync_fence(adev, &job->sync, exclusive);
1026         if (r)
1027                 goto error_free;
1028
1029         r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
1030                              owner);
1031         if (r)
1032                 goto error_free;
1033
1034         r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
1035         if (r)
1036                 goto error_free;
1037
1038         params.shadow = true;
1039         amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
1040         params.shadow = false;
1041         amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
1042
1043         amdgpu_ring_pad_ib(ring, params.ib);
1044         WARN_ON(params.ib->length_dw > ndw);
1045         r = amdgpu_job_submit(job, ring, &vm->entity,
1046                               AMDGPU_FENCE_OWNER_VM, &f);
1047         if (r)
1048                 goto error_free;
1049
1050         amdgpu_bo_fence(vm->page_directory, f, true);
1051         if (fence) {
1052                 fence_put(*fence);
1053                 *fence = fence_get(f);
1054         }
1055         fence_put(f);
1056         return 0;
1057
1058 error_free:
1059         amdgpu_job_free(job);
1060         return r;
1061 }
1062
1063 /**
1064  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1065  *
1066  * @adev: amdgpu_device pointer
1067  * @exclusive: fence we need to sync to
1068  * @gtt_flags: flags as they are used for GTT
1069  * @pages_addr: DMA addresses to use for mapping
1070  * @vm: requested vm
1071  * @mapping: mapped range and flags to use for the update
1072  * @addr: addr to set the area to
1073  * @flags: HW flags for the mapping
1074  * @fence: optional resulting fence
1075  *
1076  * Split the mapping into smaller chunks so that each update fits
1077  * into a SDMA IB.
1078  * Returns 0 for success, -EINVAL for failure.
1079  */
1080 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1081                                       struct fence *exclusive,
1082                                       uint32_t gtt_flags,
1083                                       dma_addr_t *pages_addr,
1084                                       struct amdgpu_vm *vm,
1085                                       struct amdgpu_bo_va_mapping *mapping,
1086                                       uint32_t flags, uint64_t addr,
1087                                       struct fence **fence)
1088 {
1089         const uint64_t max_size = 64ULL * 1024ULL * 1024ULL / AMDGPU_GPU_PAGE_SIZE;
1090
1091         uint64_t src = 0, start = mapping->it.start;
1092         int r;
1093
1094         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1095          * but in case of something, we filter the flags in first place
1096          */
1097         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1098                 flags &= ~AMDGPU_PTE_READABLE;
1099         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1100                 flags &= ~AMDGPU_PTE_WRITEABLE;
1101
1102         trace_amdgpu_vm_bo_update(mapping);
1103
1104         if (pages_addr) {
1105                 if (flags == gtt_flags)
1106                         src = adev->gart.table_addr + (addr >> 12) * 8;
1107                 addr = 0;
1108         }
1109         addr += mapping->offset;
1110
1111         if (!pages_addr || src)
1112                 return amdgpu_vm_bo_update_mapping(adev, exclusive,
1113                                                    src, pages_addr, vm,
1114                                                    start, mapping->it.last,
1115                                                    flags, addr, fence);
1116
1117         while (start != mapping->it.last + 1) {
1118                 uint64_t last;
1119
1120                 last = min((uint64_t)mapping->it.last, start + max_size - 1);
1121                 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1122                                                 src, pages_addr, vm,
1123                                                 start, last, flags, addr,
1124                                                 fence);
1125                 if (r)
1126                         return r;
1127
1128                 start = last + 1;
1129                 addr += max_size * AMDGPU_GPU_PAGE_SIZE;
1130         }
1131
1132         return 0;
1133 }
1134
1135 /**
1136  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1137  *
1138  * @adev: amdgpu_device pointer
1139  * @bo_va: requested BO and VM object
1140  * @clear: if true clear the entries
1141  *
1142  * Fill in the page table entries for @bo_va.
1143  * Returns 0 for success, -EINVAL for failure.
1144  */
1145 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1146                         struct amdgpu_bo_va *bo_va,
1147                         bool clear)
1148 {
1149         struct amdgpu_vm *vm = bo_va->vm;
1150         struct amdgpu_bo_va_mapping *mapping;
1151         dma_addr_t *pages_addr = NULL;
1152         uint32_t gtt_flags, flags;
1153         struct ttm_mem_reg *mem;
1154         struct fence *exclusive;
1155         uint64_t addr;
1156         int r;
1157
1158         if (clear) {
1159                 mem = NULL;
1160                 addr = 0;
1161                 exclusive = NULL;
1162         } else {
1163                 struct ttm_dma_tt *ttm;
1164
1165                 mem = &bo_va->bo->tbo.mem;
1166                 addr = (u64)mem->start << PAGE_SHIFT;
1167                 switch (mem->mem_type) {
1168                 case TTM_PL_TT:
1169                         ttm = container_of(bo_va->bo->tbo.ttm, struct
1170                                            ttm_dma_tt, ttm);
1171                         pages_addr = ttm->dma_address;
1172                         break;
1173
1174                 case TTM_PL_VRAM:
1175                         addr += adev->vm_manager.vram_base_offset;
1176                         break;
1177
1178                 default:
1179                         break;
1180                 }
1181
1182                 exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
1183         }
1184
1185         flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
1186         gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
1187                 adev == bo_va->bo->adev) ? flags : 0;
1188
1189         spin_lock(&vm->status_lock);
1190         if (!list_empty(&bo_va->vm_status))
1191                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1192         spin_unlock(&vm->status_lock);
1193
1194         list_for_each_entry(mapping, &bo_va->invalids, list) {
1195                 r = amdgpu_vm_bo_split_mapping(adev, exclusive,
1196                                                gtt_flags, pages_addr, vm,
1197                                                mapping, flags, addr,
1198                                                &bo_va->last_pt_update);
1199                 if (r)
1200                         return r;
1201         }
1202
1203         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1204                 list_for_each_entry(mapping, &bo_va->valids, list)
1205                         trace_amdgpu_vm_bo_mapping(mapping);
1206
1207                 list_for_each_entry(mapping, &bo_va->invalids, list)
1208                         trace_amdgpu_vm_bo_mapping(mapping);
1209         }
1210
1211         spin_lock(&vm->status_lock);
1212         list_splice_init(&bo_va->invalids, &bo_va->valids);
1213         list_del_init(&bo_va->vm_status);
1214         if (clear)
1215                 list_add(&bo_va->vm_status, &vm->cleared);
1216         spin_unlock(&vm->status_lock);
1217
1218         return 0;
1219 }
1220
1221 /**
1222  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1223  *
1224  * @adev: amdgpu_device pointer
1225  * @vm: requested vm
1226  *
1227  * Make sure all freed BOs are cleared in the PT.
1228  * Returns 0 for success.
1229  *
1230  * PTs have to be reserved and mutex must be locked!
1231  */
1232 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1233                           struct amdgpu_vm *vm)
1234 {
1235         struct amdgpu_bo_va_mapping *mapping;
1236         int r;
1237
1238         while (!list_empty(&vm->freed)) {
1239                 mapping = list_first_entry(&vm->freed,
1240                         struct amdgpu_bo_va_mapping, list);
1241                 list_del(&mapping->list);
1242
1243                 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
1244                                                0, 0, NULL);
1245                 kfree(mapping);
1246                 if (r)
1247                         return r;
1248
1249         }
1250         return 0;
1251
1252 }
1253
1254 /**
1255  * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1256  *
1257  * @adev: amdgpu_device pointer
1258  * @vm: requested vm
1259  *
1260  * Make sure all invalidated BOs are cleared in the PT.
1261  * Returns 0 for success.
1262  *
1263  * PTs have to be reserved and mutex must be locked!
1264  */
1265 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1266                              struct amdgpu_vm *vm, struct amdgpu_sync *sync)
1267 {
1268         struct amdgpu_bo_va *bo_va = NULL;
1269         int r = 0;
1270
1271         spin_lock(&vm->status_lock);
1272         while (!list_empty(&vm->invalidated)) {
1273                 bo_va = list_first_entry(&vm->invalidated,
1274                         struct amdgpu_bo_va, vm_status);
1275                 spin_unlock(&vm->status_lock);
1276
1277                 r = amdgpu_vm_bo_update(adev, bo_va, true);
1278                 if (r)
1279                         return r;
1280
1281                 spin_lock(&vm->status_lock);
1282         }
1283         spin_unlock(&vm->status_lock);
1284
1285         if (bo_va)
1286                 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1287
1288         return r;
1289 }
1290
1291 /**
1292  * amdgpu_vm_bo_add - add a bo to a specific vm
1293  *
1294  * @adev: amdgpu_device pointer
1295  * @vm: requested vm
1296  * @bo: amdgpu buffer object
1297  *
1298  * Add @bo into the requested vm.
1299  * Add @bo to the list of bos associated with the vm
1300  * Returns newly added bo_va or NULL for failure
1301  *
1302  * Object has to be reserved!
1303  */
1304 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1305                                       struct amdgpu_vm *vm,
1306                                       struct amdgpu_bo *bo)
1307 {
1308         struct amdgpu_bo_va *bo_va;
1309
1310         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1311         if (bo_va == NULL) {
1312                 return NULL;
1313         }
1314         bo_va->vm = vm;
1315         bo_va->bo = bo;
1316         bo_va->ref_count = 1;
1317         INIT_LIST_HEAD(&bo_va->bo_list);
1318         INIT_LIST_HEAD(&bo_va->valids);
1319         INIT_LIST_HEAD(&bo_va->invalids);
1320         INIT_LIST_HEAD(&bo_va->vm_status);
1321
1322         list_add_tail(&bo_va->bo_list, &bo->va);
1323
1324         return bo_va;
1325 }
1326
1327 /**
1328  * amdgpu_vm_bo_map - map bo inside a vm
1329  *
1330  * @adev: amdgpu_device pointer
1331  * @bo_va: bo_va to store the address
1332  * @saddr: where to map the BO
1333  * @offset: requested offset in the BO
1334  * @flags: attributes of pages (read/write/valid/etc.)
1335  *
1336  * Add a mapping of the BO at the specefied addr into the VM.
1337  * Returns 0 for success, error for failure.
1338  *
1339  * Object has to be reserved and unreserved outside!
1340  */
1341 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1342                      struct amdgpu_bo_va *bo_va,
1343                      uint64_t saddr, uint64_t offset,
1344                      uint64_t size, uint32_t flags)
1345 {
1346         struct amdgpu_bo_va_mapping *mapping;
1347         struct amdgpu_vm *vm = bo_va->vm;
1348         struct interval_tree_node *it;
1349         unsigned last_pfn, pt_idx;
1350         uint64_t eaddr;
1351         int r;
1352
1353         /* validate the parameters */
1354         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1355             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1356                 return -EINVAL;
1357
1358         /* make sure object fit at this offset */
1359         eaddr = saddr + size - 1;
1360         if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
1361                 return -EINVAL;
1362
1363         last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1364         if (last_pfn >= adev->vm_manager.max_pfn) {
1365                 dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
1366                         last_pfn, adev->vm_manager.max_pfn);
1367                 return -EINVAL;
1368         }
1369
1370         saddr /= AMDGPU_GPU_PAGE_SIZE;
1371         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1372
1373         it = interval_tree_iter_first(&vm->va, saddr, eaddr);
1374         if (it) {
1375                 struct amdgpu_bo_va_mapping *tmp;
1376                 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1377                 /* bo and tmp overlap, invalid addr */
1378                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1379                         "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1380                         tmp->it.start, tmp->it.last + 1);
1381                 r = -EINVAL;
1382                 goto error;
1383         }
1384
1385         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1386         if (!mapping) {
1387                 r = -ENOMEM;
1388                 goto error;
1389         }
1390
1391         INIT_LIST_HEAD(&mapping->list);
1392         mapping->it.start = saddr;
1393         mapping->it.last = eaddr;
1394         mapping->offset = offset;
1395         mapping->flags = flags;
1396
1397         list_add(&mapping->list, &bo_va->invalids);
1398         interval_tree_insert(&mapping->it, &vm->va);
1399
1400         /* Make sure the page tables are allocated */
1401         saddr >>= amdgpu_vm_block_size;
1402         eaddr >>= amdgpu_vm_block_size;
1403
1404         BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1405
1406         if (eaddr > vm->max_pde_used)
1407                 vm->max_pde_used = eaddr;
1408
1409         /* walk over the address space and allocate the page tables */
1410         for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1411                 struct reservation_object *resv = vm->page_directory->tbo.resv;
1412                 struct amdgpu_bo_list_entry *entry;
1413                 struct amdgpu_bo *pt;
1414
1415                 entry = &vm->page_tables[pt_idx].entry;
1416                 if (entry->robj)
1417                         continue;
1418
1419                 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1420                                      AMDGPU_GPU_PAGE_SIZE, true,
1421                                      AMDGPU_GEM_DOMAIN_VRAM,
1422                                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1423                                      AMDGPU_GEM_CREATE_SHADOW,
1424                                      NULL, resv, &pt);
1425                 if (r)
1426                         goto error_free;
1427
1428                 /* Keep a reference to the page table to avoid freeing
1429                  * them up in the wrong order.
1430                  */
1431                 pt->parent = amdgpu_bo_ref(vm->page_directory);
1432
1433                 r = amdgpu_vm_clear_bo(adev, vm, pt);
1434                 if (r) {
1435                         amdgpu_bo_unref(&pt->shadow);
1436                         amdgpu_bo_unref(&pt);
1437                         goto error_free;
1438                 }
1439
1440                 if (pt->shadow) {
1441                         r = amdgpu_vm_clear_bo(adev, vm, pt->shadow);
1442                         if (r) {
1443                                 amdgpu_bo_unref(&pt->shadow);
1444                                 amdgpu_bo_unref(&pt);
1445                                 goto error_free;
1446                         }
1447                 }
1448
1449                 entry->robj = pt;
1450                 entry->priority = 0;
1451                 entry->tv.bo = &entry->robj->tbo;
1452                 entry->tv.shared = true;
1453                 entry->user_pages = NULL;
1454                 vm->page_tables[pt_idx].addr = 0;
1455         }
1456
1457         return 0;
1458
1459 error_free:
1460         list_del(&mapping->list);
1461         interval_tree_remove(&mapping->it, &vm->va);
1462         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1463         kfree(mapping);
1464
1465 error:
1466         return r;
1467 }
1468
1469 /**
1470  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1471  *
1472  * @adev: amdgpu_device pointer
1473  * @bo_va: bo_va to remove the address from
1474  * @saddr: where to the BO is mapped
1475  *
1476  * Remove a mapping of the BO at the specefied addr from the VM.
1477  * Returns 0 for success, error for failure.
1478  *
1479  * Object has to be reserved and unreserved outside!
1480  */
1481 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1482                        struct amdgpu_bo_va *bo_va,
1483                        uint64_t saddr)
1484 {
1485         struct amdgpu_bo_va_mapping *mapping;
1486         struct amdgpu_vm *vm = bo_va->vm;
1487         bool valid = true;
1488
1489         saddr /= AMDGPU_GPU_PAGE_SIZE;
1490
1491         list_for_each_entry(mapping, &bo_va->valids, list) {
1492                 if (mapping->it.start == saddr)
1493                         break;
1494         }
1495
1496         if (&mapping->list == &bo_va->valids) {
1497                 valid = false;
1498
1499                 list_for_each_entry(mapping, &bo_va->invalids, list) {
1500                         if (mapping->it.start == saddr)
1501                                 break;
1502                 }
1503
1504                 if (&mapping->list == &bo_va->invalids)
1505                         return -ENOENT;
1506         }
1507
1508         list_del(&mapping->list);
1509         interval_tree_remove(&mapping->it, &vm->va);
1510         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1511
1512         if (valid)
1513                 list_add(&mapping->list, &vm->freed);
1514         else
1515                 kfree(mapping);
1516
1517         return 0;
1518 }
1519
1520 /**
1521  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1522  *
1523  * @adev: amdgpu_device pointer
1524  * @bo_va: requested bo_va
1525  *
1526  * Remove @bo_va->bo from the requested vm.
1527  *
1528  * Object have to be reserved!
1529  */
1530 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1531                       struct amdgpu_bo_va *bo_va)
1532 {
1533         struct amdgpu_bo_va_mapping *mapping, *next;
1534         struct amdgpu_vm *vm = bo_va->vm;
1535
1536         list_del(&bo_va->bo_list);
1537
1538         spin_lock(&vm->status_lock);
1539         list_del(&bo_va->vm_status);
1540         spin_unlock(&vm->status_lock);
1541
1542         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1543                 list_del(&mapping->list);
1544                 interval_tree_remove(&mapping->it, &vm->va);
1545                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1546                 list_add(&mapping->list, &vm->freed);
1547         }
1548         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1549                 list_del(&mapping->list);
1550                 interval_tree_remove(&mapping->it, &vm->va);
1551                 kfree(mapping);
1552         }
1553
1554         fence_put(bo_va->last_pt_update);
1555         kfree(bo_va);
1556 }
1557
1558 /**
1559  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1560  *
1561  * @adev: amdgpu_device pointer
1562  * @vm: requested vm
1563  * @bo: amdgpu buffer object
1564  *
1565  * Mark @bo as invalid.
1566  */
1567 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1568                              struct amdgpu_bo *bo)
1569 {
1570         struct amdgpu_bo_va *bo_va;
1571
1572         list_for_each_entry(bo_va, &bo->va, bo_list) {
1573                 spin_lock(&bo_va->vm->status_lock);
1574                 if (list_empty(&bo_va->vm_status))
1575                         list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1576                 spin_unlock(&bo_va->vm->status_lock);
1577         }
1578 }
1579
1580 /**
1581  * amdgpu_vm_init - initialize a vm instance
1582  *
1583  * @adev: amdgpu_device pointer
1584  * @vm: requested vm
1585  *
1586  * Init @vm fields.
1587  */
1588 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1589 {
1590         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1591                 AMDGPU_VM_PTE_COUNT * 8);
1592         unsigned pd_size, pd_entries;
1593         unsigned ring_instance;
1594         struct amdgpu_ring *ring;
1595         struct amd_sched_rq *rq;
1596         int i, r;
1597
1598         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1599                 vm->ids[i] = NULL;
1600         vm->va = RB_ROOT;
1601         vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
1602         spin_lock_init(&vm->status_lock);
1603         INIT_LIST_HEAD(&vm->invalidated);
1604         INIT_LIST_HEAD(&vm->cleared);
1605         INIT_LIST_HEAD(&vm->freed);
1606
1607         pd_size = amdgpu_vm_directory_size(adev);
1608         pd_entries = amdgpu_vm_num_pdes(adev);
1609
1610         /* allocate page table array */
1611         vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
1612         if (vm->page_tables == NULL) {
1613                 DRM_ERROR("Cannot allocate memory for page table array\n");
1614                 return -ENOMEM;
1615         }
1616
1617         /* create scheduler entity for page table updates */
1618
1619         ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
1620         ring_instance %= adev->vm_manager.vm_pte_num_rings;
1621         ring = adev->vm_manager.vm_pte_rings[ring_instance];
1622         rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
1623         r = amd_sched_entity_init(&ring->sched, &vm->entity,
1624                                   rq, amdgpu_sched_jobs);
1625         if (r)
1626                 goto err;
1627
1628         vm->page_directory_fence = NULL;
1629
1630         r = amdgpu_bo_create(adev, pd_size, align, true,
1631                              AMDGPU_GEM_DOMAIN_VRAM,
1632                              AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1633                              AMDGPU_GEM_CREATE_SHADOW,
1634                              NULL, NULL, &vm->page_directory);
1635         if (r)
1636                 goto error_free_sched_entity;
1637
1638         r = amdgpu_bo_reserve(vm->page_directory, false);
1639         if (r)
1640                 goto error_free_page_directory;
1641
1642         r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
1643         if (r)
1644                 goto error_unreserve;
1645
1646         if (vm->page_directory->shadow) {
1647                 r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory->shadow);
1648                 if (r)
1649                         goto error_unreserve;
1650         }
1651
1652         vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
1653         amdgpu_bo_unreserve(vm->page_directory);
1654
1655         return 0;
1656
1657 error_unreserve:
1658         amdgpu_bo_unreserve(vm->page_directory);
1659
1660 error_free_page_directory:
1661         amdgpu_bo_unref(&vm->page_directory->shadow);
1662         amdgpu_bo_unref(&vm->page_directory);
1663         vm->page_directory = NULL;
1664
1665 error_free_sched_entity:
1666         amd_sched_entity_fini(&ring->sched, &vm->entity);
1667
1668 err:
1669         drm_free_large(vm->page_tables);
1670
1671         return r;
1672 }
1673
1674 /**
1675  * amdgpu_vm_fini - tear down a vm instance
1676  *
1677  * @adev: amdgpu_device pointer
1678  * @vm: requested vm
1679  *
1680  * Tear down @vm.
1681  * Unbind the VM and remove all bos from the vm bo list
1682  */
1683 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1684 {
1685         struct amdgpu_bo_va_mapping *mapping, *tmp;
1686         int i;
1687
1688         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1689
1690         if (!RB_EMPTY_ROOT(&vm->va)) {
1691                 dev_err(adev->dev, "still active bo inside vm\n");
1692         }
1693         rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1694                 list_del(&mapping->list);
1695                 interval_tree_remove(&mapping->it, &vm->va);
1696                 kfree(mapping);
1697         }
1698         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1699                 list_del(&mapping->list);
1700                 kfree(mapping);
1701         }
1702
1703         for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
1704                 struct amdgpu_bo *pt = vm->page_tables[i].entry.robj;
1705
1706                 if (!pt)
1707                         continue;
1708
1709                 amdgpu_bo_unref(&pt->shadow);
1710                 amdgpu_bo_unref(&pt);
1711         }
1712         drm_free_large(vm->page_tables);
1713
1714         amdgpu_bo_unref(&vm->page_directory->shadow);
1715         amdgpu_bo_unref(&vm->page_directory);
1716         fence_put(vm->page_directory_fence);
1717 }
1718
1719 /**
1720  * amdgpu_vm_manager_init - init the VM manager
1721  *
1722  * @adev: amdgpu_device pointer
1723  *
1724  * Initialize the VM manager structures
1725  */
1726 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1727 {
1728         unsigned i;
1729
1730         INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1731
1732         /* skip over VMID 0, since it is the system VM */
1733         for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1734                 amdgpu_vm_reset_id(adev, i);
1735                 amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1736                 list_add_tail(&adev->vm_manager.ids[i].list,
1737                               &adev->vm_manager.ids_lru);
1738         }
1739
1740         adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
1741         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1742                 adev->vm_manager.seqno[i] = 0;
1743
1744         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1745         atomic64_set(&adev->vm_manager.client_counter, 0);
1746 }
1747
1748 /**
1749  * amdgpu_vm_manager_fini - cleanup VM manager
1750  *
1751  * @adev: amdgpu_device pointer
1752  *
1753  * Cleanup the VM manager and free resources.
1754  */
1755 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
1756 {
1757         unsigned i;
1758
1759         for (i = 0; i < AMDGPU_NUM_VM; ++i) {
1760                 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
1761
1762                 fence_put(adev->vm_manager.ids[i].first);
1763                 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1764                 fence_put(id->flushed_updates);
1765         }
1766 }