drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <drm/drmP.h>
  28 #include <drm/i915_drm.h>
  29 #include "i915_drv.h"
  30 #include "i915_vgpu.h"
  31 #include "i915_trace.h"
  32 #include "intel_drv.h"
  33
  34 /**
  35  * DOC: Global GTT views
  36  *
  37  * Background and previous state
  38  *
  39  * Historically objects could exists (be bound) in global GTT space only as
  40  * singular instances with a view representing all of the object's backing pages
  41  * in a linear fashion. This view will be called a normal view.
  42  *
  43  * To support multiple views of the same object, where the number of mapped
  44  * pages is not equal to the backing store, or where the layout of the pages
  45  * is not linear, concept of a GGTT view was added.
  46  *
  47  * One example of an alternative view is a stereo display driven by a single
  48  * image. In this case we would have a framebuffer looking like this
  49  * (2x2 pages):
  50  *
  51  *    12
  52  *    34
  53  *
  54  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  55  * rendering. In contrast, fed to the display engine would be an alternative
  56  * view which could look something like this:
  57  *
  58  *   1212
  59  *   3434
  60  *
  61  * In this example both the size and layout of pages in the alternative view is
  62  * different from the normal view.
  63  *
  64  * Implementation and usage
  65  *
  66  * GGTT views are implemented using VMAs and are distinguished via enum
  67  * i915_ggtt_view_type and struct i915_ggtt_view.
  68  *
  69  * A new flavour of core GEM functions which work with GGTT bound objects were
  70  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  71  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  72  * parameter encapsulating all metadata required to implement a view.
  73  *
  74  * As a helper for callers which are only interested in the normal view,
  75  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  76  * GEM API functions, the ones not taking the view parameter, are operating on,
  77  * or with the normal GGTT view.
  78  *
  79  * Code wanting to add or use a new GGTT view needs to:
  80  *
  81  * 1. Add a new enum with a suitable name.
  82  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  83  * 3. Add support to i915_get_vma_pages().
  84  *
  85  * New views are required to build a scatter-gather table from within the
  86  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  87  * exists for the lifetime of an VMA.
  88  *
  89  * Core API is designed to have copy semantics which means that passed in
  90  * struct i915_ggtt_view does not need to be persistent (left around after
  91  * calling the core API functions).
  92  *
  93  */
  94
  95 const struct i915_ggtt_view i915_ggtt_view_normal;
  96 const struct i915_ggtt_view i915_ggtt_view_rotated = {
  97         .type = I915_GGTT_VIEW_ROTATED
  98 };
  99
 100 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
 101 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
 102
 103 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 104 {
 105         bool has_aliasing_ppgtt;
 106         bool has_full_ppgtt;
 107
 108         has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
 109         has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
 110
 111         if (intel_vgpu_active(dev))
 112                 has_full_ppgtt = false; /* emulation is too hard */
 113
 114         /*
 115          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 116          * execlists, the sole mechanism available to submit work.
 117          */
 118         if (INTEL_INFO(dev)->gen < 9 &&
 119             (enable_ppgtt == 0 || !has_aliasing_ppgtt))
 120                 return 0;
 121
 122         if (enable_ppgtt == 1)
 123                 return 1;
 124
 125         if (enable_ppgtt == 2 && has_full_ppgtt)
 126                 return 2;
 127
 128 #ifdef CONFIG_INTEL_IOMMU
 129         /* Disable ppgtt on SNB if VT-d is on. */
 130         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
 131                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 132                 return 0;
 133         }
 134 #endif
 135
 136         /* Early VLV doesn't have this */
 137         if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
 138             dev->pdev->revision < 0xb) {
 139                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 140                 return 0;
 141         }
 142
 143         if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
 144                 return 2;
 145         else
 146                 return has_aliasing_ppgtt ? 1 : 0;
 147 }
 148
 149 static void ppgtt_bind_vma(struct i915_vma *vma,
 150                            enum i915_cache_level cache_level,
 151                            u32 flags);
 152 static void ppgtt_unbind_vma(struct i915_vma *vma);
 153
 154 static inline gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 155                                          enum i915_cache_level level,
 156                                          bool valid)
 157 {
 158         gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
 159         pte |= addr;
 160
 161         switch (level) {
 162         case I915_CACHE_NONE:
 163                 pte |= PPAT_UNCACHED_INDEX;
 164                 break;
 165         case I915_CACHE_WT:
 166                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 167                 break;
 168         default:
 169                 pte |= PPAT_CACHED_INDEX;
 170                 break;
 171         }
 172
 173         return pte;
 174 }
 175
 176 static inline gen8_pde_t gen8_pde_encode(struct drm_device *dev,
 177                                           dma_addr_t addr,
 178                                           enum i915_cache_level level)
 179 {
 180         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 181         pde |= addr;
 182         if (level != I915_CACHE_NONE)
 183                 pde |= PPAT_CACHED_PDE_INDEX;
 184         else
 185                 pde |= PPAT_UNCACHED_INDEX;
 186         return pde;
 187 }
 188
 189 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 190                                  enum i915_cache_level level,
 191                                  bool valid, u32 unused)
 192 {
 193         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 194         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 195
 196         switch (level) {
 197         case I915_CACHE_L3_LLC:
 198         case I915_CACHE_LLC:
 199                 pte |= GEN6_PTE_CACHE_LLC;
 200                 break;
 201         case I915_CACHE_NONE:
 202                 pte |= GEN6_PTE_UNCACHED;
 203                 break;
 204         default:
 205                 MISSING_CASE(level);
 206         }
 207
 208         return pte;
 209 }
 210
 211 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 212                                  enum i915_cache_level level,
 213                                  bool valid, u32 unused)
 214 {
 215         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 216         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 217
 218         switch (level) {
 219         case I915_CACHE_L3_LLC:
 220                 pte |= GEN7_PTE_CACHE_L3_LLC;
 221                 break;
 222         case I915_CACHE_LLC:
 223                 pte |= GEN6_PTE_CACHE_LLC;
 224                 break;
 225         case I915_CACHE_NONE:
 226                 pte |= GEN6_PTE_UNCACHED;
 227                 break;
 228         default:
 229                 MISSING_CASE(level);
 230         }
 231
 232         return pte;
 233 }
 234
 235 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 236                                  enum i915_cache_level level,
 237                                  bool valid, u32 flags)
 238 {
 239         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 240         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 241
 242         if (!(flags & PTE_READ_ONLY))
 243                 pte |= BYT_PTE_WRITEABLE;
 244
 245         if (level != I915_CACHE_NONE)
 246                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 247
 248         return pte;
 249 }
 250
 251 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 252                                  enum i915_cache_level level,
 253                                  bool valid, u32 unused)
 254 {
 255         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 256         pte |= HSW_PTE_ADDR_ENCODE(addr);
 257
 258         if (level != I915_CACHE_NONE)
 259                 pte |= HSW_WB_LLC_AGE3;
 260
 261         return pte;
 262 }
 263
 264 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 265                                   enum i915_cache_level level,
 266                                   bool valid, u32 unused)
 267 {
 268         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 269         pte |= HSW_PTE_ADDR_ENCODE(addr);
 270
 271         switch (level) {
 272         case I915_CACHE_NONE:
 273                 break;
 274         case I915_CACHE_WT:
 275                 pte |= HSW_WT_ELLC_LLC_AGE3;
 276                 break;
 277         default:
 278                 pte |= HSW_WB_ELLC_LLC_AGE3;
 279                 break;
 280         }
 281
 282         return pte;
 283 }
 284
 285 #define i915_dma_unmap_single(px, dev) \
 286         __i915_dma_unmap_single((px)->daddr, dev)
 287
 288 static inline void __i915_dma_unmap_single(dma_addr_t daddr,
 289                                         struct drm_device *dev)
 290 {
 291         struct device *device = &dev->pdev->dev;
 292
 293         dma_unmap_page(device, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 294 }
 295
 296 /**
 297  * i915_dma_map_single() - Create a dma mapping for a page table/dir/etc.
 298  * @px: Page table/dir/etc to get a DMA map for
 299  * @dev:        drm device
 300  *
 301  * Page table allocations are unified across all gens. They always require a
 302  * single 4k allocation, as well as a DMA mapping. If we keep the structs
 303  * symmetric here, the simple macro covers us for every page table type.
 304  *
 305  * Return: 0 if success.
 306  */
 307 #define i915_dma_map_single(px, dev) \
 308         i915_dma_map_page_single((px)->page, (dev), &(px)->daddr)
 309
 310 static inline int i915_dma_map_page_single(struct page *page,
 311                                            struct drm_device *dev,
 312                                            dma_addr_t *daddr)
 313 {
 314         struct device *device = &dev->pdev->dev;
 315
 316         *daddr = dma_map_page(device, page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
 317         if (dma_mapping_error(device, *daddr))
 318                 return -ENOMEM;
 319
 320         return 0;
 321 }
 322
 323 static void unmap_and_free_pt(struct i915_page_table *pt,
 324                                struct drm_device *dev)
 325 {
 326         if (WARN_ON(!pt->page))
 327                 return;
 328
 329         i915_dma_unmap_single(pt, dev);
 330         __free_page(pt->page);
 331         kfree(pt->used_ptes);
 332         kfree(pt);
 333 }
 334
 335 static void gen8_initialize_pt(struct i915_address_space *vm,
 336                                 struct i915_page_table *pt)
 337 {
 338         gen8_pte_t *pt_vaddr, scratch_pte;
 339         int i;
 340
 341         pt_vaddr = kmap_atomic(pt->page);
 342         scratch_pte = gen8_pte_encode(vm->scratch.addr,
 343                                       I915_CACHE_LLC, true);
 344
 345         for (i = 0; i < GEN8_PTES; i++)
 346                 pt_vaddr[i] = scratch_pte;
 347
 348         if (!HAS_LLC(vm->dev))
 349                 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 350         kunmap_atomic(pt_vaddr);
 351 }
 352
 353 static struct i915_page_table *alloc_pt_single(struct drm_device *dev)
 354 {
 355         struct i915_page_table *pt;
 356         const size_t count = INTEL_INFO(dev)->gen >= 8 ?
 357                 GEN8_PTES : GEN6_PTES;
 358         int ret = -ENOMEM;
 359
 360         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 361         if (!pt)
 362                 return ERR_PTR(-ENOMEM);
 363
 364         pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
 365                                 GFP_KERNEL);
 366
 367         if (!pt->used_ptes)
 368                 goto fail_bitmap;
 369
 370         pt->page = alloc_page(GFP_KERNEL);
 371         if (!pt->page)
 372                 goto fail_page;
 373
 374         ret = i915_dma_map_single(pt, dev);
 375         if (ret)
 376                 goto fail_dma;
 377
 378         return pt;
 379
 380 fail_dma:
 381         __free_page(pt->page);
 382 fail_page:
 383         kfree(pt->used_ptes);
 384 fail_bitmap:
 385         kfree(pt);
 386
 387         return ERR_PTR(ret);
 388 }
 389
 390 /**
 391  * alloc_pt_range() - Allocate a multiple page tables
 392  * @pd:         The page directory which will have at least @count entries
 393  *              available to point to the allocated page tables.
 394  * @pde:        First page directory entry for which we are allocating.
 395  * @count:      Number of pages to allocate.
 396  * @dev:        DRM device.
 397  *
 398  * Allocates multiple page table pages and sets the appropriate entries in the
 399  * page table structure within the page directory. Function cleans up after
 400  * itself on any failures.
 401  *
 402  * Return: 0 if allocation succeeded.
 403  */
 404 static int alloc_pt_range(struct i915_page_directory *pd, uint16_t pde, size_t count,
 405                           struct drm_device *dev)
 406 {
 407         int i, ret;
 408
 409         /* 512 is the max page tables per page_directory on any platform. */
 410         if (WARN_ON(pde + count > I915_PDES))
 411                 return -EINVAL;
 412
 413         for (i = pde; i < pde + count; i++) {
 414                 struct i915_page_table *pt = alloc_pt_single(dev);
 415
 416                 if (IS_ERR(pt)) {
 417                         ret = PTR_ERR(pt);
 418                         goto err_out;
 419                 }
 420                 WARN(pd->page_table[i],
 421                      "Leaking page directory entry %d (%p)\n",
 422                      i, pd->page_table[i]);
 423                 pd->page_table[i] = pt;
 424         }
 425
 426         return 0;
 427
 428 err_out:
 429         while (i-- > pde)
 430                 unmap_and_free_pt(pd->page_table[i], dev);
 431         return ret;
 432 }
 433
 434 static void unmap_and_free_pd(struct i915_page_directory *pd)
 435 {
 436         if (pd->page) {
 437                 __free_page(pd->page);
 438                 kfree(pd);
 439         }
 440 }
 441
 442 static struct i915_page_directory *alloc_pd_single(void)
 443 {
 444         struct i915_page_directory *pd;
 445
 446         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 447         if (!pd)
 448                 return ERR_PTR(-ENOMEM);
 449
 450         pd->page = alloc_page(GFP_KERNEL);
 451         if (!pd->page) {
 452                 kfree(pd);
 453                 return ERR_PTR(-ENOMEM);
 454         }
 455
 456         return pd;
 457 }
 458
 459 /* Broadwell Page Directory Pointer Descriptors */
 460 static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
 461                            uint64_t val)
 462 {
 463         int ret;
 464
 465         BUG_ON(entry >= 4);
 466
 467         ret = intel_ring_begin(ring, 6);
 468         if (ret)
 469                 return ret;
 470
 471         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 472         intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
 473         intel_ring_emit(ring, (u32)(val >> 32));
 474         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 475         intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
 476         intel_ring_emit(ring, (u32)(val));
 477         intel_ring_advance(ring);
 478
 479         return 0;
 480 }
 481
 482 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 483                           struct intel_engine_cs *ring)
 484 {
 485         int i, ret;
 486
 487         /* bit of a hack to find the actual last used pd */
 488         int used_pd = ppgtt->num_pd_entries / I915_PDES;
 489
 490         for (i = used_pd - 1; i >= 0; i--) {
 491                 dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr;
 492                 ret = gen8_write_pdp(ring, i, addr);
 493                 if (ret)
 494                         return ret;
 495         }
 496
 497         return 0;
 498 }
 499
 500 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 501                                    uint64_t start,
 502                                    uint64_t length,
 503                                    bool use_scratch)
 504 {
 505         struct i915_hw_ppgtt *ppgtt =
 506                 container_of(vm, struct i915_hw_ppgtt, base);
 507         gen8_pte_t *pt_vaddr, scratch_pte;
 508         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 509         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 510         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 511         unsigned num_entries = length >> PAGE_SHIFT;
 512         unsigned last_pte, i;
 513
 514         scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
 515                                       I915_CACHE_LLC, use_scratch);
 516
 517         while (num_entries) {
 518                 struct i915_page_directory *pd;
 519                 struct i915_page_table *pt;
 520                 struct page *page_table;
 521
 522                 if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))
 523                         continue;
 524
 525                 pd = ppgtt->pdp.page_directory[pdpe];
 526
 527                 if (WARN_ON(!pd->page_table[pde]))
 528                         continue;
 529
 530                 pt = pd->page_table[pde];
 531
 532                 if (WARN_ON(!pt->page))
 533                         continue;
 534
 535                 page_table = pt->page;
 536
 537                 last_pte = pte + num_entries;
 538                 if (last_pte > GEN8_PTES)
 539                         last_pte = GEN8_PTES;
 540
 541                 pt_vaddr = kmap_atomic(page_table);
 542
 543                 for (i = pte; i < last_pte; i++) {
 544                         pt_vaddr[i] = scratch_pte;
 545                         num_entries--;
 546                 }
 547
 548                 if (!HAS_LLC(ppgtt->base.dev))
 549                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 550                 kunmap_atomic(pt_vaddr);
 551
 552                 pte = 0;
 553                 if (++pde == I915_PDES) {
 554                         pdpe++;
 555                         pde = 0;
 556                 }
 557         }
 558 }
 559
 560 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 561                                       struct sg_table *pages,
 562                                       uint64_t start,
 563                                       enum i915_cache_level cache_level, u32 unused)
 564 {
 565         struct i915_hw_ppgtt *ppgtt =
 566                 container_of(vm, struct i915_hw_ppgtt, base);
 567         gen8_pte_t *pt_vaddr;
 568         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 569         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 570         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 571         struct sg_page_iter sg_iter;
 572
 573         pt_vaddr = NULL;
 574
 575         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 576                 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
 577                         break;
 578
 579                 if (pt_vaddr == NULL) {
 580                         struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
 581                         struct i915_page_table *pt = pd->page_table[pde];
 582                         struct page *page_table = pt->page;
 583
 584                         pt_vaddr = kmap_atomic(page_table);
 585                 }
 586
 587                 pt_vaddr[pte] =
 588                         gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
 589                                         cache_level, true);
 590                 if (++pte == GEN8_PTES) {
 591                         if (!HAS_LLC(ppgtt->base.dev))
 592                                 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 593                         kunmap_atomic(pt_vaddr);
 594                         pt_vaddr = NULL;
 595                         if (++pde == I915_PDES) {
 596                                 pdpe++;
 597                                 pde = 0;
 598                         }
 599                         pte = 0;
 600                 }
 601         }
 602         if (pt_vaddr) {
 603                 if (!HAS_LLC(ppgtt->base.dev))
 604                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 605                 kunmap_atomic(pt_vaddr);
 606         }
 607 }
 608
 609 static void __gen8_do_map_pt(gen8_pde_t * const pde,
 610                              struct i915_page_table *pt,
 611                              struct drm_device *dev)
 612 {
 613         gen8_pde_t entry =
 614                 gen8_pde_encode(dev, pt->daddr, I915_CACHE_LLC);
 615         *pde = entry;
 616 }
 617
 618 static void gen8_initialize_pd(struct i915_address_space *vm,
 619                                struct i915_page_directory *pd)
 620 {
 621         struct i915_hw_ppgtt *ppgtt =
 622                         container_of(vm, struct i915_hw_ppgtt, base);
 623         gen8_pde_t *page_directory;
 624         struct i915_page_table *pt;
 625         int i;
 626
 627         page_directory = kmap_atomic(pd->page);
 628         pt = ppgtt->scratch_pt;
 629         for (i = 0; i < I915_PDES; i++)
 630                 /* Map the PDE to the page table */
 631                 __gen8_do_map_pt(page_directory + i, pt, vm->dev);
 632
 633         if (!HAS_LLC(vm->dev))
 634                 drm_clflush_virt_range(page_directory, PAGE_SIZE);
 635
 636         kunmap_atomic(page_directory);
 637 }
 638
 639 static void gen8_free_page_tables(struct i915_page_directory *pd, struct drm_device *dev)
 640 {
 641         int i;
 642
 643         if (!pd->page)
 644                 return;
 645
 646         for (i = 0; i < I915_PDES; i++) {
 647                 if (WARN_ON(!pd->page_table[i]))
 648                         continue;
 649
 650                 unmap_and_free_pt(pd->page_table[i], dev);
 651                 pd->page_table[i] = NULL;
 652         }
 653 }
 654
 655 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 656 {
 657         int i;
 658
 659         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 660                 if (WARN_ON(!ppgtt->pdp.page_directory[i]))
 661                         continue;
 662
 663                 gen8_free_page_tables(ppgtt->pdp.page_directory[i], ppgtt->base.dev);
 664                 unmap_and_free_pd(ppgtt->pdp.page_directory[i]);
 665         }
 666
 667         unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
 668 }
 669
 670 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 671 {
 672         struct i915_hw_ppgtt *ppgtt =
 673                 container_of(vm, struct i915_hw_ppgtt, base);
 674
 675         gen8_ppgtt_free(ppgtt);
 676 }
 677
 678 static int gen8_ppgtt_alloc_pagetabs(struct i915_page_directory *pd,
 679                                      uint64_t start,
 680                                      uint64_t length,
 681                                      struct i915_address_space *vm)
 682 {
 683         struct i915_page_table *unused;
 684         uint64_t temp;
 685         uint32_t pde;
 686
 687         gen8_for_each_pde(unused, pd, start, length, temp, pde) {
 688                 WARN_ON(unused);
 689                 pd->page_table[pde] = alloc_pt_single(vm->dev);
 690                 if (IS_ERR(pd->page_table[pde]))
 691                         goto unwind_out;
 692
 693                 gen8_initialize_pt(vm, pd->page_table[pde]);
 694         }
 695
 696         /* XXX: Still alloc all page tables in systems with less than
 697          * 4GB of memory. This won't be needed after a subsequent patch.
 698          */
 699         while (pde < I915_PDES) {
 700                 pd->page_table[pde] = alloc_pt_single(vm->dev);
 701                 if (IS_ERR(pd->page_table[pde]))
 702                         goto unwind_out;
 703
 704                 gen8_initialize_pt(vm, pd->page_table[pde]);
 705                 pde++;
 706         }
 707
 708         return 0;
 709
 710 unwind_out:
 711         while (pde--)
 712                 unmap_and_free_pt(pd->page_table[pde], vm->dev);
 713
 714         return -ENOMEM;
 715 }
 716
 717 static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer *pdp,
 718                                      uint64_t start,
 719                                      uint64_t length)
 720 {
 721         struct i915_hw_ppgtt *ppgtt =
 722                 container_of(pdp, struct i915_hw_ppgtt, pdp);
 723         struct i915_page_directory *unused;
 724         uint64_t temp;
 725         uint32_t pdpe;
 726
 727         /* FIXME: PPGTT container_of won't work for 64b */
 728         WARN_ON((start + length) > 0x800000000ULL);
 729
 730         gen8_for_each_pdpe(unused, pdp, start, length, temp, pdpe) {
 731                 WARN_ON(unused);
 732                 pdp->page_directory[pdpe] = alloc_pd_single();
 733                 if (IS_ERR(ppgtt->pdp.page_directory[pdpe]))
 734                         goto unwind_out;
 735
 736                 gen8_initialize_pd(&ppgtt->base,
 737                                    ppgtt->pdp.page_directory[pdpe]);
 738                 ppgtt->num_pd_pages++;
 739         }
 740
 741         /* XXX: Still alloc all page directories in systems with less than
 742          * 4GB of memory. This won't be needed after a subsequent patch.
 743          */
 744         while (ppgtt->num_pd_pages < GEN8_LEGACY_PDPES) {
 745                 ppgtt->pdp.page_directory[ppgtt->num_pd_pages] = alloc_pd_single();
 746                 if (IS_ERR(ppgtt->pdp.page_directory[ppgtt->num_pd_pages]))
 747                         goto unwind_out;
 748
 749                 gen8_initialize_pd(&ppgtt->base,
 750                                    ppgtt->pdp.page_directory[ppgtt->num_pd_pages]);
 751                 pdpe++;
 752                 ppgtt->num_pd_pages++;
 753         }
 754
 755         BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPES);
 756
 757         return 0;
 758
 759 unwind_out:
 760         while (pdpe--) {
 761                 unmap_and_free_pd(ppgtt->pdp.page_directory[pdpe]);
 762                 ppgtt->num_pd_pages--;
 763         }
 764
 765         WARN_ON(ppgtt->num_pd_pages);
 766
 767         return -ENOMEM;
 768 }
 769
 770 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 771                             uint64_t start,
 772                             uint64_t length)
 773 {
 774         struct i915_page_directory *pd;
 775         uint64_t temp;
 776         uint32_t pdpe;
 777         int ret;
 778
 779         ret = gen8_ppgtt_alloc_page_directories(&ppgtt->pdp, start, length);
 780         if (ret)
 781                 return ret;
 782
 783         gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
 784                 ret = gen8_ppgtt_alloc_pagetabs(pd, start, length,
 785                                                 &ppgtt->base);
 786                 if (ret)
 787                         goto err_out;
 788
 789                 ppgtt->num_pd_entries += I915_PDES;
 790         }
 791
 792         /* XXX: We allocated all page directories in systems with less than
 793          * 4GB of memory. So initalize page tables of all PDPs.
 794          * This won't be needed after the next patch.
 795          */
 796         while (pdpe < GEN8_LEGACY_PDPES) {
 797                 ret = gen8_ppgtt_alloc_pagetabs(ppgtt->pdp.page_directory[pdpe], start, length,
 798                                                 &ppgtt->base);
 799                 if (ret)
 800                         goto err_out;
 801
 802                 ppgtt->num_pd_entries += I915_PDES;
 803                 pdpe++;
 804         }
 805
 806         WARN_ON(pdpe > ppgtt->num_pd_pages);
 807
 808         return 0;
 809
 810 err_out:
 811         gen8_ppgtt_free(ppgtt);
 812         return ret;
 813 }
 814
 815 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 816                                              const int pd)
 817 {
 818         dma_addr_t pd_addr;
 819         int ret;
 820
 821         pd_addr = pci_map_page(ppgtt->base.dev->pdev,
 822                                ppgtt->pdp.page_directory[pd]->page, 0,
 823                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 824
 825         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 826         if (ret)
 827                 return ret;
 828
 829         ppgtt->pdp.page_directory[pd]->daddr = pd_addr;
 830
 831         return 0;
 832 }
 833
 834 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 835                                         const int pd,
 836                                         const int pt)
 837 {
 838         dma_addr_t pt_addr;
 839         struct i915_page_directory *pdir = ppgtt->pdp.page_directory[pd];
 840         struct i915_page_table *ptab = pdir->page_table[pt];
 841         struct page *p = ptab->page;
 842         int ret;
 843
 844         gen8_initialize_pt(&ppgtt->base, ptab);
 845
 846         pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 847                                p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 848         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
 849         if (ret)
 850                 return ret;
 851
 852         ptab->daddr = pt_addr;
 853
 854         return 0;
 855 }
 856
 857 /*
 858  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 859  * with a net effect resembling a 2-level page table in normal x86 terms. Each
 860  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 861  * space.
 862  *
 863  * FIXME: split allocation into smaller pieces. For now we only ever do this
 864  * once, but with full PPGTT, the multiple contiguous allocations will be bad.
 865  * TODO: Do something with the size parameter
 866  */
 867 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 868 {
 869         const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
 870         const int min_pt_pages = I915_PDES * max_pdp;
 871         int i, j, ret;
 872
 873         if (size % (1<<30))
 874                 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
 875
 876         ppgtt->base.start = 0;
 877         ppgtt->base.total = size;
 878
 879         ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev);
 880         if (IS_ERR(ppgtt->scratch_pt))
 881                 return PTR_ERR(ppgtt->scratch_pt);
 882
 883         gen8_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);
 884
 885         /* 1. Do all our allocations for page directories and page tables. */
 886         ret = gen8_ppgtt_alloc(ppgtt, ppgtt->base.start, ppgtt->base.total);
 887         if (ret)
 888                 return ret;
 889
 890         /*
 891          * 2. Create DMA mappings for the page directories and page tables.
 892          */
 893         for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
 894                 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 895                 if (ret)
 896                         goto bail;
 897
 898                 for (j = 0; j < I915_PDES; j++) {
 899                         ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
 900                         if (ret)
 901                                 goto bail;
 902                 }
 903         }
 904
 905         /*
 906          * 3. Map all the page directory entries to point to the page tables
 907          * we've allocated.
 908          *
 909          * For now, the PPGTT helper functions all require that the PDEs are
 910          * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
 911          * will never need to touch the PDEs again.
 912          */
 913         for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
 914                 struct i915_page_directory *pd = ppgtt->pdp.page_directory[i];
 915                 gen8_pde_t *pd_vaddr;
 916                 pd_vaddr = kmap_atomic(ppgtt->pdp.page_directory[i]->page);
 917                 for (j = 0; j < I915_PDES; j++) {
 918                         struct i915_page_table *pt = pd->page_table[j];
 919                         dma_addr_t addr = pt->daddr;
 920                         pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 921                                                       I915_CACHE_LLC);
 922                 }
 923                 if (!HAS_LLC(ppgtt->base.dev))
 924                         drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
 925                 kunmap_atomic(pd_vaddr);
 926         }
 927
 928         ppgtt->switch_mm = gen8_mm_switch;
 929         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
 930         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
 931         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 932
 933         /* Set all ptes to a valid scratch page. Also above requested space */
 934         ppgtt->base.clear_range(&ppgtt->base, 0,
 935                                 ppgtt->num_pd_pages * GEN8_PTES * PAGE_SIZE,
 936                                 true);
 937
 938         DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 939                          ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
 940         DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
 941                          ppgtt->num_pd_entries,
 942                          (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
 943         return 0;
 944
 945 bail:
 946         gen8_ppgtt_free(ppgtt);
 947         return ret;
 948 }
 949
 950 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 951 {
 952         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 953         struct i915_address_space *vm = &ppgtt->base;
 954         gen6_pte_t __iomem *pd_addr;
 955         gen6_pte_t scratch_pte;
 956         uint32_t pd_entry;
 957         int pte, pde;
 958
 959         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 960
 961         pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
 962                 ppgtt->pd.pd_offset / sizeof(gen6_pte_t);
 963
 964         seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
 965                    ppgtt->pd.pd_offset,
 966                    ppgtt->pd.pd_offset + ppgtt->num_pd_entries);
 967         for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
 968                 u32 expected;
 969                 gen6_pte_t *pt_vaddr;
 970                 dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr;
 971                 pd_entry = readl(pd_addr + pde);
 972                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 973
 974                 if (pd_entry != expected)
 975                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
 976                                    pde,
 977                                    pd_entry,
 978                                    expected);
 979                 seq_printf(m, "\tPDE: %x\n", pd_entry);
 980
 981                 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page);
 982                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
 983                         unsigned long va =
 984                                 (pde * PAGE_SIZE * GEN6_PTES) +
 985                                 (pte * PAGE_SIZE);
 986                         int i;
 987                         bool found = false;
 988                         for (i = 0; i < 4; i++)
 989                                 if (pt_vaddr[pte + i] != scratch_pte)
 990                                         found = true;
 991                         if (!found)
 992                                 continue;
 993
 994                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
 995                         for (i = 0; i < 4; i++) {
 996                                 if (pt_vaddr[pte + i] != scratch_pte)
 997                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
 998                                 else
 999                                         seq_puts(m, "  SCRATCH ");
1000                         }
1001                         seq_puts(m, "\n");
1002                 }
1003                 kunmap_atomic(pt_vaddr);
1004         }
1005 }
1006
1007 /* Write pde (index) from the page directory @pd to the page table @pt */
1008 static void gen6_write_pde(struct i915_page_directory *pd,
1009                             const int pde, struct i915_page_table *pt)
1010 {
1011         /* Caller needs to make sure the write completes if necessary */
1012         struct i915_hw_ppgtt *ppgtt =
1013                 container_of(pd, struct i915_hw_ppgtt, pd);
1014         u32 pd_entry;
1015
1016         pd_entry = GEN6_PDE_ADDR_ENCODE(pt->daddr);
1017         pd_entry |= GEN6_PDE_VALID;
1018
1019         writel(pd_entry, ppgtt->pd_addr + pde);
1020 }
1021
1022 /* Write all the page tables found in the ppgtt structure to incrementing page
1023  * directories. */
1024 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1025                                   struct i915_page_directory *pd,
1026                                   uint32_t start, uint32_t length)
1027 {
1028         struct i915_page_table *pt;
1029         uint32_t pde, temp;
1030
1031         gen6_for_each_pde(pt, pd, start, length, temp, pde)
1032                 gen6_write_pde(pd, pde, pt);
1033
1034         /* Make sure write is complete before other code can use this page
1035          * table. Also require for WC mapped PTEs */
1036         readl(dev_priv->gtt.gsm);
1037 }
1038
1039 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1040 {
1041         BUG_ON(ppgtt->pd.pd_offset & 0x3f);
1042
1043         return (ppgtt->pd.pd_offset / 64) << 16;
1044 }
1045
1046 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1047                          struct intel_engine_cs *ring)
1048 {
1049         int ret;
1050
1051         /* NB: TLBs must be flushed and invalidated before a switch */
1052         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1053         if (ret)
1054                 return ret;
1055
1056         ret = intel_ring_begin(ring, 6);
1057         if (ret)
1058                 return ret;
1059
1060         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1061         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1062         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1063         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1064         intel_ring_emit(ring, get_pd_offset(ppgtt));
1065         intel_ring_emit(ring, MI_NOOP);
1066         intel_ring_advance(ring);
1067
1068         return 0;
1069 }
1070
1071 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
1072                           struct intel_engine_cs *ring)
1073 {
1074         struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1075
1076         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1077         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1078         return 0;
1079 }
1080
1081 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1082                           struct intel_engine_cs *ring)
1083 {
1084         int ret;
1085
1086         /* NB: TLBs must be flushed and invalidated before a switch */
1087         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1088         if (ret)
1089                 return ret;
1090
1091         ret = intel_ring_begin(ring, 6);
1092         if (ret)
1093                 return ret;
1094
1095         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1096         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1097         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1098         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1099         intel_ring_emit(ring, get_pd_offset(ppgtt));
1100         intel_ring_emit(ring, MI_NOOP);
1101         intel_ring_advance(ring);
1102
1103         /* XXX: RCS is the only one to auto invalidate the TLBs? */
1104         if (ring->id != RCS) {
1105                 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1106                 if (ret)
1107                         return ret;
1108         }
1109
1110         return 0;
1111 }
1112
1113 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1114                           struct intel_engine_cs *ring)
1115 {
1116         struct drm_device *dev = ppgtt->base.dev;
1117         struct drm_i915_private *dev_priv = dev->dev_private;
1118
1119
1120         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1121         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1122
1123         POSTING_READ(RING_PP_DIR_DCLV(ring));
1124
1125         return 0;
1126 }
1127
1128 static void gen8_ppgtt_enable(struct drm_device *dev)
1129 {
1130         struct drm_i915_private *dev_priv = dev->dev_private;
1131         struct intel_engine_cs *ring;
1132         int j;
1133
1134         for_each_ring(ring, dev_priv, j) {
1135                 I915_WRITE(RING_MODE_GEN7(ring),
1136                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1137         }
1138 }
1139
1140 static void gen7_ppgtt_enable(struct drm_device *dev)
1141 {
1142         struct drm_i915_private *dev_priv = dev->dev_private;
1143         struct intel_engine_cs *ring;
1144         uint32_t ecochk, ecobits;
1145         int i;
1146
1147         ecobits = I915_READ(GAC_ECO_BITS);
1148         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1149
1150         ecochk = I915_READ(GAM_ECOCHK);
1151         if (IS_HASWELL(dev)) {
1152                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1153         } else {
1154                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1155                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1156         }
1157         I915_WRITE(GAM_ECOCHK, ecochk);
1158
1159         for_each_ring(ring, dev_priv, i) {
1160                 /* GFX_MODE is per-ring on gen7+ */
1161                 I915_WRITE(RING_MODE_GEN7(ring),
1162                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1163         }
1164 }
1165
1166 static void gen6_ppgtt_enable(struct drm_device *dev)
1167 {
1168         struct drm_i915_private *dev_priv = dev->dev_private;
1169         uint32_t ecochk, gab_ctl, ecobits;
1170
1171         ecobits = I915_READ(GAC_ECO_BITS);
1172         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1173                    ECOBITS_PPGTT_CACHE64B);
1174
1175         gab_ctl = I915_READ(GAB_CTL);
1176         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1177
1178         ecochk = I915_READ(GAM_ECOCHK);
1179         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1180
1181         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1182 }
1183
1184 /* PPGTT support for Sandybdrige/Gen6 and later */
1185 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1186                                    uint64_t start,
1187                                    uint64_t length,
1188                                    bool use_scratch)
1189 {
1190         struct i915_hw_ppgtt *ppgtt =
1191                 container_of(vm, struct i915_hw_ppgtt, base);
1192         gen6_pte_t *pt_vaddr, scratch_pte;
1193         unsigned first_entry = start >> PAGE_SHIFT;
1194         unsigned num_entries = length >> PAGE_SHIFT;
1195         unsigned act_pt = first_entry / GEN6_PTES;
1196         unsigned first_pte = first_entry % GEN6_PTES;
1197         unsigned last_pte, i;
1198
1199         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
1200
1201         while (num_entries) {
1202                 last_pte = first_pte + num_entries;
1203                 if (last_pte > GEN6_PTES)
1204                         last_pte = GEN6_PTES;
1205
1206                 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
1207
1208                 for (i = first_pte; i < last_pte; i++)
1209                         pt_vaddr[i] = scratch_pte;
1210
1211                 kunmap_atomic(pt_vaddr);
1212
1213                 num_entries -= last_pte - first_pte;
1214                 first_pte = 0;
1215                 act_pt++;
1216         }
1217 }
1218
1219 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1220                                       struct sg_table *pages,
1221                                       uint64_t start,
1222                                       enum i915_cache_level cache_level, u32 flags)
1223 {
1224         struct i915_hw_ppgtt *ppgtt =
1225                 container_of(vm, struct i915_hw_ppgtt, base);
1226         gen6_pte_t *pt_vaddr;
1227         unsigned first_entry = start >> PAGE_SHIFT;
1228         unsigned act_pt = first_entry / GEN6_PTES;
1229         unsigned act_pte = first_entry % GEN6_PTES;
1230         struct sg_page_iter sg_iter;
1231
1232         pt_vaddr = NULL;
1233         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
1234                 if (pt_vaddr == NULL)
1235                         pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
1236
1237                 pt_vaddr[act_pte] =
1238                         vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
1239                                        cache_level, true, flags);
1240
1241                 if (++act_pte == GEN6_PTES) {
1242                         kunmap_atomic(pt_vaddr);
1243                         pt_vaddr = NULL;
1244                         act_pt++;
1245                         act_pte = 0;
1246                 }
1247         }
1248         if (pt_vaddr)
1249                 kunmap_atomic(pt_vaddr);
1250 }
1251
1252 /* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we
1253  * are switching between contexts with the same LRCA, we also must do a force
1254  * restore.
1255  */
1256 static inline void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
1257 {
1258         /* If current vm != vm, */
1259         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
1260 }
1261
1262 static void gen6_initialize_pt(struct i915_address_space *vm,
1263                 struct i915_page_table *pt)
1264 {
1265         gen6_pte_t *pt_vaddr, scratch_pte;
1266         int i;
1267
1268         WARN_ON(vm->scratch.addr == 0);
1269
1270         scratch_pte = vm->pte_encode(vm->scratch.addr,
1271                         I915_CACHE_LLC, true, 0);
1272
1273         pt_vaddr = kmap_atomic(pt->page);
1274
1275         for (i = 0; i < GEN6_PTES; i++)
1276                 pt_vaddr[i] = scratch_pte;
1277
1278         kunmap_atomic(pt_vaddr);
1279 }
1280
1281 static int gen6_alloc_va_range(struct i915_address_space *vm,
1282                                uint64_t start, uint64_t length)
1283 {
1284         DECLARE_BITMAP(new_page_tables, I915_PDES);
1285         struct drm_device *dev = vm->dev;
1286         struct drm_i915_private *dev_priv = dev->dev_private;
1287         struct i915_hw_ppgtt *ppgtt =
1288                                 container_of(vm, struct i915_hw_ppgtt, base);
1289         struct i915_page_table *pt;
1290         const uint32_t start_save = start, length_save = length;
1291         uint32_t pde, temp;
1292         int ret;
1293
1294         WARN_ON(upper_32_bits(start));
1295
1296         bitmap_zero(new_page_tables, I915_PDES);
1297
1298         /* The allocation is done in two stages so that we can bail out with
1299          * minimal amount of pain. The first stage finds new page tables that
1300          * need allocation. The second stage marks use ptes within the page
1301          * tables.
1302          */
1303         gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1304                 if (pt != ppgtt->scratch_pt) {
1305                         WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1306                         continue;
1307                 }
1308
1309                 /* We've already allocated a page table */
1310                 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1311
1312                 pt = alloc_pt_single(dev);
1313                 if (IS_ERR(pt)) {
1314                         ret = PTR_ERR(pt);
1315                         goto unwind_out;
1316                 }
1317
1318                 gen6_initialize_pt(vm, pt);
1319
1320                 ppgtt->pd.page_table[pde] = pt;
1321                 set_bit(pde, new_page_tables);
1322                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1323         }
1324
1325         start = start_save;
1326         length = length_save;
1327
1328         gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1329                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1330
1331                 bitmap_zero(tmp_bitmap, GEN6_PTES);
1332                 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1333                            gen6_pte_count(start, length));
1334
1335                 if (test_and_clear_bit(pde, new_page_tables))
1336                         gen6_write_pde(&ppgtt->pd, pde, pt);
1337
1338                 trace_i915_page_table_entry_map(vm, pde, pt,
1339                                          gen6_pte_index(start),
1340                                          gen6_pte_count(start, length),
1341                                          GEN6_PTES);
1342                 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
1343                                 GEN6_PTES);
1344         }
1345
1346         WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1347
1348         /* Make sure write is complete before other code can use this page
1349          * table. Also require for WC mapped PTEs */
1350         readl(dev_priv->gtt.gsm);
1351
1352         mark_tlbs_dirty(ppgtt);
1353         return 0;
1354
1355 unwind_out:
1356         for_each_set_bit(pde, new_page_tables, I915_PDES) {
1357                 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
1358
1359                 ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;
1360                 unmap_and_free_pt(pt, vm->dev);
1361         }
1362
1363         mark_tlbs_dirty(ppgtt);
1364         return ret;
1365 }
1366
1367 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
1368 {
1369         int i;
1370
1371         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1372                 struct i915_page_table *pt = ppgtt->pd.page_table[i];
1373
1374                 if (pt != ppgtt->scratch_pt)
1375                         unmap_and_free_pt(ppgtt->pd.page_table[i], ppgtt->base.dev);
1376         }
1377
1378         unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
1379         unmap_and_free_pd(&ppgtt->pd);
1380 }
1381
1382 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1383 {
1384         struct i915_hw_ppgtt *ppgtt =
1385                 container_of(vm, struct i915_hw_ppgtt, base);
1386
1387         drm_mm_remove_node(&ppgtt->node);
1388
1389         gen6_ppgtt_free(ppgtt);
1390 }
1391
1392 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1393 {
1394         struct drm_device *dev = ppgtt->base.dev;
1395         struct drm_i915_private *dev_priv = dev->dev_private;
1396         bool retried = false;
1397         int ret;
1398
1399         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1400          * allocator works in address space sizes, so it's multiplied by page
1401          * size. We allocate at the top of the GTT to avoid fragmentation.
1402          */
1403         BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
1404         ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev);
1405         if (IS_ERR(ppgtt->scratch_pt))
1406                 return PTR_ERR(ppgtt->scratch_pt);
1407
1408         gen6_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);
1409
1410 alloc:
1411         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1412                                                   &ppgtt->node, GEN6_PD_SIZE,
1413                                                   GEN6_PD_ALIGN, 0,
1414                                                   0, dev_priv->gtt.base.total,
1415                                                   DRM_MM_TOPDOWN);
1416         if (ret == -ENOSPC && !retried) {
1417                 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1418                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
1419                                                I915_CACHE_NONE,
1420                                                0, dev_priv->gtt.base.total,
1421                                                0);
1422                 if (ret)
1423                         goto err_out;
1424
1425                 retried = true;
1426                 goto alloc;
1427         }
1428
1429         if (ret)
1430                 goto err_out;
1431
1432
1433         if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1434                 DRM_DEBUG("Forced to use aperture for PDEs\n");
1435
1436         ppgtt->num_pd_entries = I915_PDES;
1437         return 0;
1438
1439 err_out:
1440         unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
1441         return ret;
1442 }
1443
1444 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1445 {
1446         return gen6_ppgtt_allocate_page_directories(ppgtt);
1447 }
1448
1449 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
1450                                   uint64_t start, uint64_t length)
1451 {
1452         struct i915_page_table *unused;
1453         uint32_t pde, temp;
1454
1455         gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
1456                 ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;
1457 }
1458
1459 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)
1460 {
1461         struct drm_device *dev = ppgtt->base.dev;
1462         struct drm_i915_private *dev_priv = dev->dev_private;
1463         int ret;
1464
1465         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1466         if (IS_GEN6(dev)) {
1467                 ppgtt->switch_mm = gen6_mm_switch;
1468         } else if (IS_HASWELL(dev)) {
1469                 ppgtt->switch_mm = hsw_mm_switch;
1470         } else if (IS_GEN7(dev)) {
1471                 ppgtt->switch_mm = gen7_mm_switch;
1472         } else
1473                 BUG();
1474
1475         if (intel_vgpu_active(dev))
1476                 ppgtt->switch_mm = vgpu_mm_switch;
1477
1478         ret = gen6_ppgtt_alloc(ppgtt);
1479         if (ret)
1480                 return ret;
1481
1482         if (aliasing) {
1483                 /* preallocate all pts */
1484                 ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,
1485                                 ppgtt->base.dev);
1486
1487                 if (ret) {
1488                         gen6_ppgtt_cleanup(&ppgtt->base);
1489                         return ret;
1490                 }
1491         }
1492
1493         ppgtt->base.allocate_va_range = gen6_alloc_va_range;
1494         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1495         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1496         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1497         ppgtt->base.start = 0;
1498         ppgtt->base.total = ppgtt->num_pd_entries * GEN6_PTES * PAGE_SIZE;
1499         ppgtt->debug_dump = gen6_dump_ppgtt;
1500
1501         ppgtt->pd.pd_offset =
1502                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
1503
1504         ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
1505                 ppgtt->pd.pd_offset / sizeof(gen6_pte_t);
1506
1507         if (aliasing)
1508                 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1509         else
1510                 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
1511
1512         gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
1513
1514         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
1515                          ppgtt->node.size >> 20,
1516                          ppgtt->node.start / PAGE_SIZE);
1517
1518         DRM_DEBUG("Adding PPGTT at offset %x\n",
1519                   ppgtt->pd.pd_offset << 10);
1520
1521         return 0;
1522 }
1523
1524 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,
1525                 bool aliasing)
1526 {
1527         struct drm_i915_private *dev_priv = dev->dev_private;
1528
1529         ppgtt->base.dev = dev;
1530         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
1531
1532         if (INTEL_INFO(dev)->gen < 8)
1533                 return gen6_ppgtt_init(ppgtt, aliasing);
1534         else
1535                 return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1536 }
1537 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1538 {
1539         struct drm_i915_private *dev_priv = dev->dev_private;
1540         int ret = 0;
1541
1542         ret = __hw_ppgtt_init(dev, ppgtt, false);
1543         if (ret == 0) {
1544                 kref_init(&ppgtt->ref);
1545                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1546                             ppgtt->base.total);
1547                 i915_init_vm(dev_priv, &ppgtt->base);
1548         }
1549
1550         return ret;
1551 }
1552
1553 int i915_ppgtt_init_hw(struct drm_device *dev)
1554 {
1555         struct drm_i915_private *dev_priv = dev->dev_private;
1556         struct intel_engine_cs *ring;
1557         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1558         int i, ret = 0;
1559
1560         /* In the case of execlists, PPGTT is enabled by the context descriptor
1561          * and the PDPs are contained within the context itself.  We don't
1562          * need to do anything here. */
1563         if (i915.enable_execlists)
1564                 return 0;
1565
1566         if (!USES_PPGTT(dev))
1567                 return 0;
1568
1569         if (IS_GEN6(dev))
1570                 gen6_ppgtt_enable(dev);
1571         else if (IS_GEN7(dev))
1572                 gen7_ppgtt_enable(dev);
1573         else if (INTEL_INFO(dev)->gen >= 8)
1574                 gen8_ppgtt_enable(dev);
1575         else
1576                 MISSING_CASE(INTEL_INFO(dev)->gen);
1577
1578         if (ppgtt) {
1579                 for_each_ring(ring, dev_priv, i) {
1580                         ret = ppgtt->switch_mm(ppgtt, ring);
1581                         if (ret != 0)
1582                                 return ret;
1583                 }
1584         }
1585
1586         return ret;
1587 }
1588 struct i915_hw_ppgtt *
1589 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1590 {
1591         struct i915_hw_ppgtt *ppgtt;
1592         int ret;
1593
1594         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1595         if (!ppgtt)
1596                 return ERR_PTR(-ENOMEM);
1597
1598         ret = i915_ppgtt_init(dev, ppgtt);
1599         if (ret) {
1600                 kfree(ppgtt);
1601                 return ERR_PTR(ret);
1602         }
1603
1604         ppgtt->file_priv = fpriv;
1605
1606         trace_i915_ppgtt_create(&ppgtt->base);
1607
1608         return ppgtt;
1609 }
1610
1611 void  i915_ppgtt_release(struct kref *kref)
1612 {
1613         struct i915_hw_ppgtt *ppgtt =
1614                 container_of(kref, struct i915_hw_ppgtt, ref);
1615
1616         trace_i915_ppgtt_release(&ppgtt->base);
1617
1618         /* vmas should already be unbound */
1619         WARN_ON(!list_empty(&ppgtt->base.active_list));
1620         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1621
1622         list_del(&ppgtt->base.global_link);
1623         drm_mm_takedown(&ppgtt->base.mm);
1624
1625         ppgtt->base.cleanup(&ppgtt->base);
1626         kfree(ppgtt);
1627 }
1628
1629 static void
1630 ppgtt_bind_vma(struct i915_vma *vma,
1631                enum i915_cache_level cache_level,
1632                u32 flags)
1633 {
1634         /* Currently applicable only to VLV */
1635         if (vma->obj->gt_ro)
1636                 flags |= PTE_READ_ONLY;
1637
1638         vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1639                                 cache_level, flags);
1640 }
1641
1642 static void ppgtt_unbind_vma(struct i915_vma *vma)
1643 {
1644         vma->vm->clear_range(vma->vm,
1645                              vma->node.start,
1646                              vma->obj->base.size,
1647                              true);
1648 }
1649
1650 extern int intel_iommu_gfx_mapped;
1651 /* Certain Gen5 chipsets require require idling the GPU before
1652  * unmapping anything from the GTT when VT-d is enabled.
1653  */
1654 static inline bool needs_idle_maps(struct drm_device *dev)
1655 {
1656 #ifdef CONFIG_INTEL_IOMMU
1657         /* Query intel_iommu to see if we need the workaround. Presumably that
1658          * was loaded first.
1659          */
1660         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1661                 return true;
1662 #endif
1663         return false;
1664 }
1665
1666 static bool do_idling(struct drm_i915_private *dev_priv)
1667 {
1668         bool ret = dev_priv->mm.interruptible;
1669
1670         if (unlikely(dev_priv->gtt.do_idle_maps)) {
1671                 dev_priv->mm.interruptible = false;
1672                 if (i915_gpu_idle(dev_priv->dev)) {
1673                         DRM_ERROR("Couldn't idle GPU\n");
1674                         /* Wait a bit, in hopes it avoids the hang */
1675                         udelay(10);
1676                 }
1677         }
1678
1679         return ret;
1680 }
1681
1682 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1683 {
1684         if (unlikely(dev_priv->gtt.do_idle_maps))
1685                 dev_priv->mm.interruptible = interruptible;
1686 }
1687
1688 void i915_check_and_clear_faults(struct drm_device *dev)
1689 {
1690         struct drm_i915_private *dev_priv = dev->dev_private;
1691         struct intel_engine_cs *ring;
1692         int i;
1693
1694         if (INTEL_INFO(dev)->gen < 6)
1695                 return;
1696
1697         for_each_ring(ring, dev_priv, i) {
1698                 u32 fault_reg;
1699                 fault_reg = I915_READ(RING_FAULT_REG(ring));
1700                 if (fault_reg & RING_FAULT_VALID) {
1701                         DRM_DEBUG_DRIVER("Unexpected fault\n"
1702                                          "\tAddr: 0x%08lx\n"
1703                                          "\tAddress space: %s\n"
1704                                          "\tSource ID: %d\n"
1705                                          "\tType: %d\n",
1706                                          fault_reg & PAGE_MASK,
1707                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1708                                          RING_FAULT_SRCID(fault_reg),
1709                                          RING_FAULT_FAULT_TYPE(fault_reg));
1710                         I915_WRITE(RING_FAULT_REG(ring),
1711                                    fault_reg & ~RING_FAULT_VALID);
1712                 }
1713         }
1714         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1715 }
1716
1717 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1718 {
1719         if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1720                 intel_gtt_chipset_flush();
1721         } else {
1722                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1723                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1724         }
1725 }
1726
1727 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1728 {
1729         struct drm_i915_private *dev_priv = dev->dev_private;
1730
1731         /* Don't bother messing with faults pre GEN6 as we have little
1732          * documentation supporting that it's a good idea.
1733          */
1734         if (INTEL_INFO(dev)->gen < 6)
1735                 return;
1736
1737         i915_check_and_clear_faults(dev);
1738
1739         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1740                                        dev_priv->gtt.base.start,
1741                                        dev_priv->gtt.base.total,
1742                                        true);
1743
1744         i915_ggtt_flush(dev_priv);
1745 }
1746
1747 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1748 {
1749         struct drm_i915_private *dev_priv = dev->dev_private;
1750         struct drm_i915_gem_object *obj;
1751         struct i915_address_space *vm;
1752
1753         i915_check_and_clear_faults(dev);
1754
1755         /* First fill our portion of the GTT with scratch pages */
1756         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1757                                        dev_priv->gtt.base.start,
1758                                        dev_priv->gtt.base.total,
1759                                        true);
1760
1761         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1762                 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1763                                                            &dev_priv->gtt.base);
1764                 if (!vma)
1765                         continue;
1766
1767                 i915_gem_clflush_object(obj, obj->pin_display);
1768                 /* The bind_vma code tries to be smart about tracking mappings.
1769                  * Unfortunately above, we've just wiped out the mappings
1770                  * without telling our object about it. So we need to fake it.
1771                  *
1772                  * Bind is not expected to fail since this is only called on
1773                  * resume and assumption is all requirements exist already.
1774                  */
1775                 vma->bound &= ~GLOBAL_BIND;
1776                 WARN_ON(i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND));
1777         }
1778
1779
1780         if (INTEL_INFO(dev)->gen >= 8) {
1781                 if (IS_CHERRYVIEW(dev))
1782                         chv_setup_private_ppat(dev_priv);
1783                 else
1784                         bdw_setup_private_ppat(dev_priv);
1785
1786                 return;
1787         }
1788
1789         if (USES_PPGTT(dev)) {
1790                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1791                         /* TODO: Perhaps it shouldn't be gen6 specific */
1792
1793                         struct i915_hw_ppgtt *ppgtt =
1794                                         container_of(vm, struct i915_hw_ppgtt,
1795                                                      base);
1796
1797                         if (i915_is_ggtt(vm))
1798                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
1799
1800                         gen6_write_page_range(dev_priv, &ppgtt->pd,
1801                                               0, ppgtt->base.total);
1802                 }
1803         }
1804
1805         i915_ggtt_flush(dev_priv);
1806 }
1807
1808 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1809 {
1810         if (obj->has_dma_mapping)
1811                 return 0;
1812
1813         if (!dma_map_sg(&obj->base.dev->pdev->dev,
1814                         obj->pages->sgl, obj->pages->nents,
1815                         PCI_DMA_BIDIRECTIONAL))
1816                 return -ENOSPC;
1817
1818         return 0;
1819 }
1820
1821 static inline void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
1822 {
1823 #ifdef writeq
1824         writeq(pte, addr);
1825 #else
1826         iowrite32((u32)pte, addr);
1827         iowrite32(pte >> 32, addr + 4);
1828 #endif
1829 }
1830
1831 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1832                                      struct sg_table *st,
1833                                      uint64_t start,
1834                                      enum i915_cache_level level, u32 unused)
1835 {
1836         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1837         unsigned first_entry = start >> PAGE_SHIFT;
1838         gen8_pte_t __iomem *gtt_entries =
1839                 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1840         int i = 0;
1841         struct sg_page_iter sg_iter;
1842         dma_addr_t addr = 0; /* shut up gcc */
1843
1844         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1845                 addr = sg_dma_address(sg_iter.sg) +
1846                         (sg_iter.sg_pgoffset << PAGE_SHIFT);
1847                 gen8_set_pte(&gtt_entries[i],
1848                              gen8_pte_encode(addr, level, true));
1849                 i++;
1850         }
1851
1852         /*
1853          * XXX: This serves as a posting read to make sure that the PTE has
1854          * actually been updated. There is some concern that even though
1855          * registers and PTEs are within the same BAR that they are potentially
1856          * of NUMA access patterns. Therefore, even with the way we assume
1857          * hardware should work, we must keep this posting read for paranoia.
1858          */
1859         if (i != 0)
1860                 WARN_ON(readq(&gtt_entries[i-1])
1861                         != gen8_pte_encode(addr, level, true));
1862
1863         /* This next bit makes the above posting read even more important. We
1864          * want to flush the TLBs only after we're certain all the PTE updates
1865          * have finished.
1866          */
1867         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1868         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1869 }
1870
1871 /*
1872  * Binds an object into the global gtt with the specified cache level. The object
1873  * will be accessible to the GPU via commands whose operands reference offsets
1874  * within the global GTT as well as accessible by the GPU through the GMADR
1875  * mapped BAR (dev_priv->mm.gtt->gtt).
1876  */
1877 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1878                                      struct sg_table *st,
1879                                      uint64_t start,
1880                                      enum i915_cache_level level, u32 flags)
1881 {
1882         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1883         unsigned first_entry = start >> PAGE_SHIFT;
1884         gen6_pte_t __iomem *gtt_entries =
1885                 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1886         int i = 0;
1887         struct sg_page_iter sg_iter;
1888         dma_addr_t addr = 0;
1889
1890         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1891                 addr = sg_page_iter_dma_address(&sg_iter);
1892                 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
1893                 i++;
1894         }
1895
1896         /* XXX: This serves as a posting read to make sure that the PTE has
1897          * actually been updated. There is some concern that even though
1898          * registers and PTEs are within the same BAR that they are potentially
1899          * of NUMA access patterns. Therefore, even with the way we assume
1900          * hardware should work, we must keep this posting read for paranoia.
1901          */
1902         if (i != 0) {
1903                 unsigned long gtt = readl(&gtt_entries[i-1]);
1904                 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1905         }
1906
1907         /* This next bit makes the above posting read even more important. We
1908          * want to flush the TLBs only after we're certain all the PTE updates
1909          * have finished.
1910          */
1911         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1912         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1913 }
1914
1915 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1916                                   uint64_t start,
1917                                   uint64_t length,
1918                                   bool use_scratch)
1919 {
1920         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1921         unsigned first_entry = start >> PAGE_SHIFT;
1922         unsigned num_entries = length >> PAGE_SHIFT;
1923         gen8_pte_t scratch_pte, __iomem *gtt_base =
1924                 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1925         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1926         int i;
1927
1928         if (WARN(num_entries > max_entries,
1929                  "First entry = %d; Num entries = %d (max=%d)\n",
1930                  first_entry, num_entries, max_entries))
1931                 num_entries = max_entries;
1932
1933         scratch_pte = gen8_pte_encode(vm->scratch.addr,
1934                                       I915_CACHE_LLC,
1935                                       use_scratch);
1936         for (i = 0; i < num_entries; i++)
1937                 gen8_set_pte(&gtt_base[i], scratch_pte);
1938         readl(gtt_base);
1939 }
1940
1941 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1942                                   uint64_t start,
1943                                   uint64_t length,
1944                                   bool use_scratch)
1945 {
1946         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1947         unsigned first_entry = start >> PAGE_SHIFT;
1948         unsigned num_entries = length >> PAGE_SHIFT;
1949         gen6_pte_t scratch_pte, __iomem *gtt_base =
1950                 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1951         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1952         int i;
1953
1954         if (WARN(num_entries > max_entries,
1955                  "First entry = %d; Num entries = %d (max=%d)\n",
1956                  first_entry, num_entries, max_entries))
1957                 num_entries = max_entries;
1958
1959         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
1960
1961         for (i = 0; i < num_entries; i++)
1962                 iowrite32(scratch_pte, &gtt_base[i]);
1963         readl(gtt_base);
1964 }
1965
1966
1967 static void i915_ggtt_bind_vma(struct i915_vma *vma,
1968                                enum i915_cache_level cache_level,
1969                                u32 unused)
1970 {
1971         const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1972         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1973                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1974
1975         BUG_ON(!i915_is_ggtt(vma->vm));
1976         intel_gtt_insert_sg_entries(vma->ggtt_view.pages, entry, flags);
1977         vma->bound = GLOBAL_BIND;
1978 }
1979
1980 static void i915_ggtt_clear_range(struct i915_address_space *vm,
1981                                   uint64_t start,
1982                                   uint64_t length,
1983                                   bool unused)
1984 {
1985         unsigned first_entry = start >> PAGE_SHIFT;
1986         unsigned num_entries = length >> PAGE_SHIFT;
1987         intel_gtt_clear_range(first_entry, num_entries);
1988 }
1989
1990 static void i915_ggtt_unbind_vma(struct i915_vma *vma)
1991 {
1992         const unsigned int first = vma->node.start >> PAGE_SHIFT;
1993         const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1994
1995         BUG_ON(!i915_is_ggtt(vma->vm));
1996         vma->bound = 0;
1997         intel_gtt_clear_range(first, size);
1998 }
1999
2000 static void ggtt_bind_vma(struct i915_vma *vma,
2001                           enum i915_cache_level cache_level,
2002                           u32 flags)
2003 {
2004         struct drm_device *dev = vma->vm->dev;
2005         struct drm_i915_private *dev_priv = dev->dev_private;
2006         struct drm_i915_gem_object *obj = vma->obj;
2007         struct sg_table *pages = obj->pages;
2008
2009         /* Currently applicable only to VLV */
2010         if (obj->gt_ro)
2011                 flags |= PTE_READ_ONLY;
2012
2013         if (i915_is_ggtt(vma->vm))
2014                 pages = vma->ggtt_view.pages;
2015
2016         /* If there is no aliasing PPGTT, or the caller needs a global mapping,
2017          * or we have a global mapping already but the cacheability flags have
2018          * changed, set the global PTEs.
2019          *
2020          * If there is an aliasing PPGTT it is anecdotally faster, so use that
2021          * instead if none of the above hold true.
2022          *
2023          * NB: A global mapping should only be needed for special regions like
2024          * "gtt mappable", SNB errata, or if specified via special execbuf
2025          * flags. At all other times, the GPU will use the aliasing PPGTT.
2026          */
2027         if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
2028                 if (!(vma->bound & GLOBAL_BIND) ||
2029                     (cache_level != obj->cache_level)) {
2030                         vma->vm->insert_entries(vma->vm, pages,
2031                                                 vma->node.start,
2032                                                 cache_level, flags);
2033                         vma->bound |= GLOBAL_BIND;
2034                 }
2035         }
2036
2037         if (dev_priv->mm.aliasing_ppgtt &&
2038             (!(vma->bound & LOCAL_BIND) ||
2039              (cache_level != obj->cache_level))) {
2040                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2041                 appgtt->base.insert_entries(&appgtt->base, pages,
2042                                             vma->node.start,
2043                                             cache_level, flags);
2044                 vma->bound |= LOCAL_BIND;
2045         }
2046 }
2047
2048 static void ggtt_unbind_vma(struct i915_vma *vma)
2049 {
2050         struct drm_device *dev = vma->vm->dev;
2051         struct drm_i915_private *dev_priv = dev->dev_private;
2052         struct drm_i915_gem_object *obj = vma->obj;
2053
2054         if (vma->bound & GLOBAL_BIND) {
2055                 vma->vm->clear_range(vma->vm,
2056                                      vma->node.start,
2057                                      obj->base.size,
2058                                      true);
2059                 vma->bound &= ~GLOBAL_BIND;
2060         }
2061
2062         if (vma->bound & LOCAL_BIND) {
2063                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2064                 appgtt->base.clear_range(&appgtt->base,
2065                                          vma->node.start,
2066                                          obj->base.size,
2067                                          true);
2068                 vma->bound &= ~LOCAL_BIND;
2069         }
2070 }
2071
2072 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
2073 {
2074         struct drm_device *dev = obj->base.dev;
2075         struct drm_i915_private *dev_priv = dev->dev_private;
2076         bool interruptible;
2077
2078         interruptible = do_idling(dev_priv);
2079
2080         if (!obj->has_dma_mapping)
2081                 dma_unmap_sg(&dev->pdev->dev,
2082                              obj->pages->sgl, obj->pages->nents,
2083                              PCI_DMA_BIDIRECTIONAL);
2084
2085         undo_idling(dev_priv, interruptible);
2086 }
2087
2088 static void i915_gtt_color_adjust(struct drm_mm_node *node,
2089                                   unsigned long color,
2090                                   u64 *start,
2091                                   u64 *end)
2092 {
2093         if (node->color != color)
2094                 *start += 4096;
2095
2096         if (!list_empty(&node->node_list)) {
2097                 node = list_entry(node->node_list.next,
2098                                   struct drm_mm_node,
2099                                   node_list);
2100                 if (node->allocated && node->color != color)
2101                         *end -= 4096;
2102         }
2103 }
2104
2105 static int i915_gem_setup_global_gtt(struct drm_device *dev,
2106                                      unsigned long start,
2107                                      unsigned long mappable_end,
2108                                      unsigned long end)
2109 {
2110         /* Let GEM Manage all of the aperture.
2111          *
2112          * However, leave one page at the end still bound to the scratch page.
2113          * There are a number of places where the hardware apparently prefetches
2114          * past the end of the object, and we've seen multiple hangs with the
2115          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2116          * aperture.  One page should be enough to keep any prefetching inside
2117          * of the aperture.
2118          */
2119         struct drm_i915_private *dev_priv = dev->dev_private;
2120         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
2121         struct drm_mm_node *entry;
2122         struct drm_i915_gem_object *obj;
2123         unsigned long hole_start, hole_end;
2124         int ret;
2125
2126         BUG_ON(mappable_end > end);
2127
2128         /* Subtract the guard page ... */
2129         drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
2130
2131         dev_priv->gtt.base.start = start;
2132         dev_priv->gtt.base.total = end - start;
2133
2134         if (intel_vgpu_active(dev)) {
2135                 ret = intel_vgt_balloon(dev);
2136                 if (ret)
2137                         return ret;
2138         }
2139
2140         if (!HAS_LLC(dev))
2141                 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
2142
2143         /* Mark any preallocated objects as occupied */
2144         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2145                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2146
2147                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
2148                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
2149
2150                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
2151                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
2152                 if (ret) {
2153                         DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
2154                         return ret;
2155                 }
2156                 vma->bound |= GLOBAL_BIND;
2157         }
2158
2159         /* Clear any non-preallocated blocks */
2160         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
2161                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2162                               hole_start, hole_end);
2163                 ggtt_vm->clear_range(ggtt_vm, hole_start,
2164                                      hole_end - hole_start, true);
2165         }
2166
2167         /* And finally clear the reserved guard page */
2168         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
2169
2170         if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
2171                 struct i915_hw_ppgtt *ppgtt;
2172
2173                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2174                 if (!ppgtt)
2175                         return -ENOMEM;
2176
2177                 ret = __hw_ppgtt_init(dev, ppgtt, true);
2178                 if (ret) {
2179                         kfree(ppgtt);
2180                         return ret;
2181                 }
2182
2183                 dev_priv->mm.aliasing_ppgtt = ppgtt;
2184         }
2185
2186         return 0;
2187 }
2188
2189 void i915_gem_init_global_gtt(struct drm_device *dev)
2190 {
2191         struct drm_i915_private *dev_priv = dev->dev_private;
2192         unsigned long gtt_size, mappable_size;
2193
2194         gtt_size = dev_priv->gtt.base.total;
2195         mappable_size = dev_priv->gtt.mappable_end;
2196
2197         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
2198 }
2199
2200 void i915_global_gtt_cleanup(struct drm_device *dev)
2201 {
2202         struct drm_i915_private *dev_priv = dev->dev_private;
2203         struct i915_address_space *vm = &dev_priv->gtt.base;
2204
2205         if (dev_priv->mm.aliasing_ppgtt) {
2206                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2207
2208                 ppgtt->base.cleanup(&ppgtt->base);
2209         }
2210
2211         if (drm_mm_initialized(&vm->mm)) {
2212                 if (intel_vgpu_active(dev))
2213                         intel_vgt_deballoon();
2214
2215                 drm_mm_takedown(&vm->mm);
2216                 list_del(&vm->global_link);
2217         }
2218
2219         vm->cleanup(vm);
2220 }
2221
2222 static int setup_scratch_page(struct drm_device *dev)
2223 {
2224         struct drm_i915_private *dev_priv = dev->dev_private;
2225         struct page *page;
2226         dma_addr_t dma_addr;
2227
2228         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
2229         if (page == NULL)
2230                 return -ENOMEM;
2231         set_pages_uc(page, 1);
2232
2233 #ifdef CONFIG_INTEL_IOMMU
2234         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
2235                                 PCI_DMA_BIDIRECTIONAL);
2236         if (pci_dma_mapping_error(dev->pdev, dma_addr))
2237                 return -EINVAL;
2238 #else
2239         dma_addr = page_to_phys(page);
2240 #endif
2241         dev_priv->gtt.base.scratch.page = page;
2242         dev_priv->gtt.base.scratch.addr = dma_addr;
2243
2244         return 0;
2245 }
2246
2247 static void teardown_scratch_page(struct drm_device *dev)
2248 {
2249         struct drm_i915_private *dev_priv = dev->dev_private;
2250         struct page *page = dev_priv->gtt.base.scratch.page;
2251
2252         set_pages_wb(page, 1);
2253         pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
2254                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
2255         __free_page(page);
2256 }
2257
2258 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2259 {
2260         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2261         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2262         return snb_gmch_ctl << 20;
2263 }
2264
2265 static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2266 {
2267         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2268         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2269         if (bdw_gmch_ctl)
2270                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2271
2272 #ifdef CONFIG_X86_32
2273         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2274         if (bdw_gmch_ctl > 4)
2275                 bdw_gmch_ctl = 4;
2276 #endif
2277
2278         return bdw_gmch_ctl << 20;
2279 }
2280
2281 static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2282 {
2283         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2284         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2285
2286         if (gmch_ctrl)
2287                 return 1 << (20 + gmch_ctrl);
2288
2289         return 0;
2290 }
2291
2292 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2293 {
2294         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2295         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2296         return snb_gmch_ctl << 25; /* 32 MB units */
2297 }
2298
2299 static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2300 {
2301         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2302         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2303         return bdw_gmch_ctl << 25; /* 32 MB units */
2304 }
2305
2306 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2307 {
2308         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2309         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2310
2311         /*
2312          * 0x0  to 0x10: 32MB increments starting at 0MB
2313          * 0x11 to 0x16: 4MB increments starting at 8MB
2314          * 0x17 to 0x1d: 4MB increments start at 36MB
2315          */
2316         if (gmch_ctrl < 0x11)
2317                 return gmch_ctrl << 25;
2318         else if (gmch_ctrl < 0x17)
2319                 return (gmch_ctrl - 0x11 + 2) << 22;
2320         else
2321                 return (gmch_ctrl - 0x17 + 9) << 22;
2322 }
2323
2324 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2325 {
2326         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2327         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2328
2329         if (gen9_gmch_ctl < 0xf0)
2330                 return gen9_gmch_ctl << 25; /* 32 MB units */
2331         else
2332                 /* 4MB increments starting at 0xf0 for 4MB */
2333                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2334 }
2335
2336 static int ggtt_probe_common(struct drm_device *dev,
2337                              size_t gtt_size)
2338 {
2339         struct drm_i915_private *dev_priv = dev->dev_private;
2340         phys_addr_t gtt_phys_addr;
2341         int ret;
2342
2343         /* For Modern GENs the PTEs and register space are split in the BAR */
2344         gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
2345                 (pci_resource_len(dev->pdev, 0) / 2);
2346
2347         dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
2348         if (!dev_priv->gtt.gsm) {
2349                 DRM_ERROR("Failed to map the gtt page table\n");
2350                 return -ENOMEM;
2351         }
2352
2353         ret = setup_scratch_page(dev);
2354         if (ret) {
2355                 DRM_ERROR("Scratch setup failed\n");
2356                 /* iounmap will also get called at remove, but meh */
2357                 iounmap(dev_priv->gtt.gsm);
2358         }
2359
2360         return ret;
2361 }
2362
2363 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2364  * bits. When using advanced contexts each context stores its own PAT, but
2365  * writing this data shouldn't be harmful even in those cases. */
2366 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2367 {
2368         uint64_t pat;
2369
2370         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2371               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2372               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2373               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2374               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2375               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2376               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2377               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2378
2379         if (!USES_PPGTT(dev_priv->dev))
2380                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2381                  * so RTL will always use the value corresponding to
2382                  * pat_sel = 000".
2383                  * So let's disable cache for GGTT to avoid screen corruptions.
2384                  * MOCS still can be used though.
2385                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2386                  * before this patch, i.e. the same uncached + snooping access
2387                  * like on gen6/7 seems to be in effect.
2388                  * - So this just fixes blitter/render access. Again it looks
2389                  * like it's not just uncached access, but uncached + snooping.
2390                  * So we can still hold onto all our assumptions wrt cpu
2391                  * clflushing on LLC machines.
2392                  */
2393                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2394
2395         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2396          * write would work. */
2397         I915_WRITE(GEN8_PRIVATE_PAT, pat);
2398         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2399 }
2400
2401 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2402 {
2403         uint64_t pat;
2404
2405         /*
2406          * Map WB on BDW to snooped on CHV.
2407          *
2408          * Only the snoop bit has meaning for CHV, the rest is
2409          * ignored.
2410          *
2411          * The hardware will never snoop for certain types of accesses:
2412          * - CPU GTT (GMADR->GGTT->no snoop->memory)
2413          * - PPGTT page tables
2414          * - some other special cycles
2415          *
2416          * As with BDW, we also need to consider the following for GT accesses:
2417          * "For GGTT, there is NO pat_sel[2:0] from the entry,
2418          * so RTL will always use the value corresponding to
2419          * pat_sel = 000".
2420          * Which means we must set the snoop bit in PAT entry 0
2421          * in order to keep the global status page working.
2422          */
2423         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2424               GEN8_PPAT(1, 0) |
2425               GEN8_PPAT(2, 0) |
2426               GEN8_PPAT(3, 0) |
2427               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2428               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2429               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2430               GEN8_PPAT(7, CHV_PPAT_SNOOP);
2431
2432         I915_WRITE(GEN8_PRIVATE_PAT, pat);
2433         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2434 }
2435
2436 static int gen8_gmch_probe(struct drm_device *dev,
2437                            size_t *gtt_total,
2438                            size_t *stolen,
2439                            phys_addr_t *mappable_base,
2440                            unsigned long *mappable_end)
2441 {
2442         struct drm_i915_private *dev_priv = dev->dev_private;
2443         unsigned int gtt_size;
2444         u16 snb_gmch_ctl;
2445         int ret;
2446
2447         /* TODO: We're not aware of mappable constraints on gen8 yet */
2448         *mappable_base = pci_resource_start(dev->pdev, 2);
2449         *mappable_end = pci_resource_len(dev->pdev, 2);
2450
2451         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2452                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2453
2454         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2455
2456         if (INTEL_INFO(dev)->gen >= 9) {
2457                 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
2458                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2459         } else if (IS_CHERRYVIEW(dev)) {
2460                 *stolen = chv_get_stolen_size(snb_gmch_ctl);
2461                 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
2462         } else {
2463                 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
2464                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2465         }
2466
2467         *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
2468
2469         if (IS_CHERRYVIEW(dev))
2470                 chv_setup_private_ppat(dev_priv);
2471         else
2472                 bdw_setup_private_ppat(dev_priv);
2473
2474         ret = ggtt_probe_common(dev, gtt_size);
2475
2476         dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
2477         dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
2478
2479         return ret;
2480 }
2481
2482 static int gen6_gmch_probe(struct drm_device *dev,
2483                            size_t *gtt_total,
2484                            size_t *stolen,
2485                            phys_addr_t *mappable_base,
2486                            unsigned long *mappable_end)
2487 {
2488         struct drm_i915_private *dev_priv = dev->dev_private;
2489         unsigned int gtt_size;
2490         u16 snb_gmch_ctl;
2491         int ret;
2492
2493         *mappable_base = pci_resource_start(dev->pdev, 2);
2494         *mappable_end = pci_resource_len(dev->pdev, 2);
2495
2496         /* 64/512MB is the current min/max we actually know of, but this is just
2497          * a coarse sanity check.
2498          */
2499         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
2500                 DRM_ERROR("Unknown GMADR size (%lx)\n",
2501                           dev_priv->gtt.mappable_end);
2502                 return -ENXIO;
2503         }
2504
2505         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2506                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
2507         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2508
2509         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
2510
2511         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
2512         *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
2513
2514         ret = ggtt_probe_common(dev, gtt_size);
2515
2516         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2517         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
2518
2519         return ret;
2520 }
2521
2522 static void gen6_gmch_remove(struct i915_address_space *vm)
2523 {
2524
2525         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
2526
2527         iounmap(gtt->gsm);
2528         teardown_scratch_page(vm->dev);
2529 }
2530
2531 static int i915_gmch_probe(struct drm_device *dev,
2532                            size_t *gtt_total,
2533                            size_t *stolen,
2534                            phys_addr_t *mappable_base,
2535                            unsigned long *mappable_end)
2536 {
2537         struct drm_i915_private *dev_priv = dev->dev_private;
2538         int ret;
2539
2540         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2541         if (!ret) {
2542                 DRM_ERROR("failed to set up gmch\n");
2543                 return -EIO;
2544         }
2545
2546         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
2547
2548         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
2549         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
2550
2551         if (unlikely(dev_priv->gtt.do_idle_maps))
2552                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2553
2554         return 0;
2555 }
2556
2557 static void i915_gmch_remove(struct i915_address_space *vm)
2558 {
2559         intel_gmch_remove();
2560 }
2561
2562 int i915_gem_gtt_init(struct drm_device *dev)
2563 {
2564         struct drm_i915_private *dev_priv = dev->dev_private;
2565         struct i915_gtt *gtt = &dev_priv->gtt;
2566         int ret;
2567
2568         if (INTEL_INFO(dev)->gen <= 5) {
2569                 gtt->gtt_probe = i915_gmch_probe;
2570                 gtt->base.cleanup = i915_gmch_remove;
2571         } else if (INTEL_INFO(dev)->gen < 8) {
2572                 gtt->gtt_probe = gen6_gmch_probe;
2573                 gtt->base.cleanup = gen6_gmch_remove;
2574                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
2575                         gtt->base.pte_encode = iris_pte_encode;
2576                 else if (IS_HASWELL(dev))
2577                         gtt->base.pte_encode = hsw_pte_encode;
2578                 else if (IS_VALLEYVIEW(dev))
2579                         gtt->base.pte_encode = byt_pte_encode;
2580                 else if (INTEL_INFO(dev)->gen >= 7)
2581                         gtt->base.pte_encode = ivb_pte_encode;
2582                 else
2583                         gtt->base.pte_encode = snb_pte_encode;
2584         } else {
2585                 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2586                 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
2587         }
2588
2589         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
2590                              &gtt->mappable_base, &gtt->mappable_end);
2591         if (ret)
2592                 return ret;
2593
2594         gtt->base.dev = dev;
2595
2596         /* GMADR is the PCI mmio aperture into the global GTT. */
2597         DRM_INFO("Memory usable by graphics device = %zdM\n",
2598                  gtt->base.total >> 20);
2599         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2600         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2601 #ifdef CONFIG_INTEL_IOMMU
2602         if (intel_iommu_gfx_mapped)
2603                 DRM_INFO("VT-d active for gfx access\n");
2604 #endif
2605         /*
2606          * i915.enable_ppgtt is read-only, so do an early pass to validate the
2607          * user's requested state against the hardware/driver capabilities.  We
2608          * do this now so that we can print out any log messages once rather
2609          * than every time we check intel_enable_ppgtt().
2610          */
2611         i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2612         DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
2613
2614         return 0;
2615 }
2616
2617 static struct i915_vma *
2618 __i915_gem_vma_create(struct drm_i915_gem_object *obj,
2619                       struct i915_address_space *vm,
2620                       const struct i915_ggtt_view *ggtt_view)
2621 {
2622         struct i915_vma *vma;
2623
2624         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
2625                 return ERR_PTR(-EINVAL);
2626         vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2627         if (vma == NULL)
2628                 return ERR_PTR(-ENOMEM);
2629
2630         INIT_LIST_HEAD(&vma->vma_link);
2631         INIT_LIST_HEAD(&vma->mm_list);
2632         INIT_LIST_HEAD(&vma->exec_list);
2633         vma->vm = vm;
2634         vma->obj = obj;
2635
2636         if (INTEL_INFO(vm->dev)->gen >= 6) {
2637                 if (i915_is_ggtt(vm)) {
2638                         vma->ggtt_view = *ggtt_view;
2639
2640                         vma->unbind_vma = ggtt_unbind_vma;
2641                         vma->bind_vma = ggtt_bind_vma;
2642                 } else {
2643                         vma->unbind_vma = ppgtt_unbind_vma;
2644                         vma->bind_vma = ppgtt_bind_vma;
2645                 }
2646         } else {
2647                 BUG_ON(!i915_is_ggtt(vm));
2648                 vma->ggtt_view = *ggtt_view;
2649                 vma->unbind_vma = i915_ggtt_unbind_vma;
2650                 vma->bind_vma = i915_ggtt_bind_vma;
2651         }
2652
2653         list_add_tail(&vma->vma_link, &obj->vma_list);
2654         if (!i915_is_ggtt(vm))
2655                 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
2656
2657         return vma;
2658 }
2659
2660 struct i915_vma *
2661 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
2662                                   struct i915_address_space *vm)
2663 {
2664         struct i915_vma *vma;
2665
2666         vma = i915_gem_obj_to_vma(obj, vm);
2667         if (!vma)
2668                 vma = __i915_gem_vma_create(obj, vm,
2669                                             i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
2670
2671         return vma;
2672 }
2673
2674 struct i915_vma *
2675 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
2676                                        const struct i915_ggtt_view *view)
2677 {
2678         struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
2679         struct i915_vma *vma;
2680
2681         if (WARN_ON(!view))
2682                 return ERR_PTR(-EINVAL);
2683
2684         vma = i915_gem_obj_to_ggtt_view(obj, view);
2685
2686         if (IS_ERR(vma))
2687                 return vma;
2688
2689         if (!vma)
2690                 vma = __i915_gem_vma_create(obj, ggtt, view);
2691
2692         return vma;
2693
2694 }
2695
2696 static void
2697 rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height,
2698              struct sg_table *st)
2699 {
2700         unsigned int column, row;
2701         unsigned int src_idx;
2702         struct scatterlist *sg = st->sgl;
2703
2704         st->nents = 0;
2705
2706         for (column = 0; column < width; column++) {
2707                 src_idx = width * (height - 1) + column;
2708                 for (row = 0; row < height; row++) {
2709                         st->nents++;
2710                         /* We don't need the pages, but need to initialize
2711                          * the entries so the sg list can be happily traversed.
2712                          * The only thing we need are DMA addresses.
2713                          */
2714                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
2715                         sg_dma_address(sg) = in[src_idx];
2716                         sg_dma_len(sg) = PAGE_SIZE;
2717                         sg = sg_next(sg);
2718                         src_idx -= width;
2719                 }
2720         }
2721 }
2722
2723 static struct sg_table *
2724 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
2725                           struct drm_i915_gem_object *obj)
2726 {
2727         struct drm_device *dev = obj->base.dev;
2728         struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
2729         unsigned long size, pages, rot_pages;
2730         struct sg_page_iter sg_iter;
2731         unsigned long i;
2732         dma_addr_t *page_addr_list;
2733         struct sg_table *st;
2734         unsigned int tile_pitch, tile_height;
2735         unsigned int width_pages, height_pages;
2736         int ret = -ENOMEM;
2737
2738         pages = obj->base.size / PAGE_SIZE;
2739
2740         /* Calculate tiling geometry. */
2741         tile_height = intel_tile_height(dev, rot_info->pixel_format,
2742                                         rot_info->fb_modifier);
2743         tile_pitch = PAGE_SIZE / tile_height;
2744         width_pages = DIV_ROUND_UP(rot_info->pitch, tile_pitch);
2745         height_pages = DIV_ROUND_UP(rot_info->height, tile_height);
2746         rot_pages = width_pages * height_pages;
2747         size = rot_pages * PAGE_SIZE;
2748
2749         /* Allocate a temporary list of source pages for random access. */
2750         page_addr_list = drm_malloc_ab(pages, sizeof(dma_addr_t));
2751         if (!page_addr_list)
2752                 return ERR_PTR(ret);
2753
2754         /* Allocate target SG list. */
2755         st = kmalloc(sizeof(*st), GFP_KERNEL);
2756         if (!st)
2757                 goto err_st_alloc;
2758
2759         ret = sg_alloc_table(st, rot_pages, GFP_KERNEL);
2760         if (ret)
2761                 goto err_sg_alloc;
2762
2763         /* Populate source page list from the object. */
2764         i = 0;
2765         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2766                 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
2767                 i++;
2768         }
2769
2770         /* Rotate the pages. */
2771         rotate_pages(page_addr_list, width_pages, height_pages, st);
2772
2773         DRM_DEBUG_KMS(
2774                       "Created rotated page mapping for object size %lu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages).\n",
2775                       size, rot_info->pitch, rot_info->height,
2776                       rot_info->pixel_format, width_pages, height_pages,
2777                       rot_pages);
2778
2779         drm_free_large(page_addr_list);
2780
2781         return st;
2782
2783 err_sg_alloc:
2784         kfree(st);
2785 err_st_alloc:
2786         drm_free_large(page_addr_list);
2787
2788         DRM_DEBUG_KMS(
2789                       "Failed to create rotated mapping for object size %lu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages)\n",
2790                       size, ret, rot_info->pitch, rot_info->height,
2791                       rot_info->pixel_format, width_pages, height_pages,
2792                       rot_pages);
2793         return ERR_PTR(ret);
2794 }
2795
2796 static inline int
2797 i915_get_ggtt_vma_pages(struct i915_vma *vma)
2798 {
2799         int ret = 0;
2800
2801         if (vma->ggtt_view.pages)
2802                 return 0;
2803
2804         if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
2805                 vma->ggtt_view.pages = vma->obj->pages;
2806         else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
2807                 vma->ggtt_view.pages =
2808                         intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
2809         else
2810                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
2811                           vma->ggtt_view.type);
2812
2813         if (!vma->ggtt_view.pages) {
2814                 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
2815                           vma->ggtt_view.type);
2816                 ret = -EINVAL;
2817         } else if (IS_ERR(vma->ggtt_view.pages)) {
2818                 ret = PTR_ERR(vma->ggtt_view.pages);
2819                 vma->ggtt_view.pages = NULL;
2820                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
2821                           vma->ggtt_view.type, ret);
2822         }
2823
2824         return ret;
2825 }
2826
2827 /**
2828  * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
2829  * @vma: VMA to map
2830  * @cache_level: mapping cache level
2831  * @flags: flags like global or local mapping
2832  *
2833  * DMA addresses are taken from the scatter-gather table of this object (or of
2834  * this VMA in case of non-default GGTT views) and PTE entries set up.
2835  * Note that DMA addresses are also the only part of the SG table we care about.
2836  */
2837 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
2838                   u32 flags)
2839 {
2840         if (i915_is_ggtt(vma->vm)) {
2841                 int ret = i915_get_ggtt_vma_pages(vma);
2842
2843                 if (ret)
2844                         return ret;
2845         }
2846
2847         vma->bind_vma(vma, cache_level, flags);
2848
2849         return 0;
2850 }