drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186
 187 /*
 188  * 0: Present
 189  * 1-11: Reserved
 190  * 12-63: Context Ptr (12 - (haw-1))
 191  * 64-127: Reserved
 192  */
 193 struct root_entry {
 194         u64     lo;
 195         u64     hi;
 196 };
 197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 198
 199 /*
 200  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 201  * if marked present.
 202  */
 203 static phys_addr_t root_entry_lctp(struct root_entry *re)
 204 {
 205         if (!(re->lo & 1))
 206                 return 0;
 207
 208         return re->lo & VTD_PAGE_MASK;
 209 }
 210
 211 /*
 212  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 213  * if marked present.
 214  */
 215 static phys_addr_t root_entry_uctp(struct root_entry *re)
 216 {
 217         if (!(re->hi & 1))
 218                 return 0;
 219
 220         return re->hi & VTD_PAGE_MASK;
 221 }
 222 /*
 223  * low 64 bits:
 224  * 0: present
 225  * 1: fault processing disable
 226  * 2-3: translation type
 227  * 12-63: address space root
 228  * high 64 bits:
 229  * 0-2: address width
 230  * 3-6: aval
 231  * 8-23: domain id
 232  */
 233 struct context_entry {
 234         u64 lo;
 235         u64 hi;
 236 };
 237
 238 static inline void context_clear_pasid_enable(struct context_entry *context)
 239 {
 240         context->lo &= ~(1ULL << 11);
 241 }
 242
 243 static inline bool context_pasid_enabled(struct context_entry *context)
 244 {
 245         return !!(context->lo & (1ULL << 11));
 246 }
 247
 248 static inline void context_set_copied(struct context_entry *context)
 249 {
 250         context->hi |= (1ull << 3);
 251 }
 252
 253 static inline bool context_copied(struct context_entry *context)
 254 {
 255         return !!(context->hi & (1ULL << 3));
 256 }
 257
 258 static inline bool __context_present(struct context_entry *context)
 259 {
 260         return (context->lo & 1);
 261 }
 262
 263 static inline bool context_present(struct context_entry *context)
 264 {
 265         return context_pasid_enabled(context) ?
 266              __context_present(context) :
 267              __context_present(context) && !context_copied(context);
 268 }
 269
 270 static inline void context_set_present(struct context_entry *context)
 271 {
 272         context->lo |= 1;
 273 }
 274
 275 static inline void context_set_fault_enable(struct context_entry *context)
 276 {
 277         context->lo &= (((u64)-1) << 2) | 1;
 278 }
 279
 280 static inline void context_set_translation_type(struct context_entry *context,
 281                                                 unsigned long value)
 282 {
 283         context->lo &= (((u64)-1) << 4) | 3;
 284         context->lo |= (value & 3) << 2;
 285 }
 286
 287 static inline void context_set_address_root(struct context_entry *context,
 288                                             unsigned long value)
 289 {
 290         context->lo &= ~VTD_PAGE_MASK;
 291         context->lo |= value & VTD_PAGE_MASK;
 292 }
 293
 294 static inline void context_set_address_width(struct context_entry *context,
 295                                              unsigned long value)
 296 {
 297         context->hi |= value & 7;
 298 }
 299
 300 static inline void context_set_domain_id(struct context_entry *context,
 301                                          unsigned long value)
 302 {
 303         context->hi |= (value & ((1 << 16) - 1)) << 8;
 304 }
 305
 306 static inline int context_domain_id(struct context_entry *c)
 307 {
 308         return((c->hi >> 8) & 0xffff);
 309 }
 310
 311 static inline void context_clear_entry(struct context_entry *context)
 312 {
 313         context->lo = 0;
 314         context->hi = 0;
 315 }
 316
 317 /*
 318  * 0: readable
 319  * 1: writable
 320  * 2-6: reserved
 321  * 7: super page
 322  * 8-10: available
 323  * 11: snoop behavior
 324  * 12-63: Host physcial address
 325  */
 326 struct dma_pte {
 327         u64 val;
 328 };
 329
 330 static inline void dma_clear_pte(struct dma_pte *pte)
 331 {
 332         pte->val = 0;
 333 }
 334
 335 static inline u64 dma_pte_addr(struct dma_pte *pte)
 336 {
 337 #ifdef CONFIG_64BIT
 338         return pte->val & VTD_PAGE_MASK;
 339 #else
 340         /* Must have a full atomic 64-bit read */
 341         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 342 #endif
 343 }
 344
 345 static inline bool dma_pte_present(struct dma_pte *pte)
 346 {
 347         return (pte->val & 3) != 0;
 348 }
 349
 350 static inline bool dma_pte_superpage(struct dma_pte *pte)
 351 {
 352         return (pte->val & DMA_PTE_LARGE_PAGE);
 353 }
 354
 355 static inline int first_pte_in_page(struct dma_pte *pte)
 356 {
 357         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 358 }
 359
 360 /*
 361  * This domain is a statically identity mapping domain.
 362  *      1. This domain creats a static 1:1 mapping to all usable memory.
 363  *      2. It maps to each iommu if successful.
 364  *      3. Each iommu mapps to this domain if successful.
 365  */
 366 static struct dmar_domain *si_domain;
 367 static int hw_pass_through = 1;
 368
 369 /*
 370  * Domain represents a virtual machine, more than one devices
 371  * across iommus may be owned in one domain, e.g. kvm guest.
 372  */
 373 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 374
 375 /* si_domain contains mulitple devices */
 376 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 377
 378 #define for_each_domain_iommu(idx, domain)                      \
 379         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 380                 if (domain->iommu_refcnt[idx])
 381
 382 struct dmar_domain {
 383         int     nid;                    /* node id */
 384
 385         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 386                                         /* Refcount of devices per iommu */
 387
 388
 389         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 390                                         /* Domain ids per IOMMU. Use u16 since
 391                                          * domain ids are 16 bit wide according
 392                                          * to VT-d spec, section 9.3 */
 393
 394         bool has_iotlb_device;
 395         struct list_head devices;       /* all devices' list */
 396         struct iova_domain iovad;       /* iova's that belong to this domain */
 397
 398         struct dma_pte  *pgd;           /* virtual address */
 399         int             gaw;            /* max guest address width */
 400
 401         /* adjusted guest address width, 0 is level 2 30-bit */
 402         int             agaw;
 403
 404         int             flags;          /* flags to find out type of domain */
 405
 406         int             iommu_coherency;/* indicate coherency of iommu access */
 407         int             iommu_snooping; /* indicate snooping control feature*/
 408         int             iommu_count;    /* reference count of iommu */
 409         int             iommu_superpage;/* Level of superpages supported:
 410                                            0 == 4KiB (no superpages), 1 == 2MiB,
 411                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 412         u64             max_addr;       /* maximum mapped address */
 413
 414         struct iommu_domain domain;     /* generic domain data structure for
 415                                            iommu core */
 416 };
 417
 418 /* PCI domain-device relationship */
 419 struct device_domain_info {
 420         struct list_head link;  /* link to domain siblings */
 421         struct list_head global; /* link to global list */
 422         u8 bus;                 /* PCI bus number */
 423         u8 devfn;               /* PCI devfn number */
 424         u8 pasid_supported:3;
 425         u8 pasid_enabled:1;
 426         u8 pri_supported:1;
 427         u8 pri_enabled:1;
 428         u8 ats_supported:1;
 429         u8 ats_enabled:1;
 430         u8 ats_qdep;
 431         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 432         struct intel_iommu *iommu; /* IOMMU used by this device */
 433         struct dmar_domain *domain; /* pointer to domain */
 434 };
 435
 436 struct dmar_rmrr_unit {
 437         struct list_head list;          /* list of rmrr units   */
 438         struct acpi_dmar_header *hdr;   /* ACPI header          */
 439         u64     base_address;           /* reserved base address*/
 440         u64     end_address;            /* reserved end address */
 441         struct dmar_dev_scope *devices; /* target devices */
 442         int     devices_cnt;            /* target device count */
 443 };
 444
 445 struct dmar_atsr_unit {
 446         struct list_head list;          /* list of ATSR units */
 447         struct acpi_dmar_header *hdr;   /* ACPI header */
 448         struct dmar_dev_scope *devices; /* target devices */
 449         int devices_cnt;                /* target device count */
 450         u8 include_all:1;               /* include all ports */
 451 };
 452
 453 static LIST_HEAD(dmar_atsr_units);
 454 static LIST_HEAD(dmar_rmrr_units);
 455
 456 #define for_each_rmrr_units(rmrr) \
 457         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 458
 459 static void flush_unmaps_timeout(unsigned long data);
 460
 461 struct deferred_flush_entry {
 462         unsigned long iova_pfn;
 463         unsigned long nrpages;
 464         struct dmar_domain *domain;
 465         struct page *freelist;
 466 };
 467
 468 #define HIGH_WATER_MARK 250
 469 struct deferred_flush_table {
 470         int next;
 471         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 472 };
 473
 474 struct deferred_flush_data {
 475         spinlock_t lock;
 476         int timer_on;
 477         struct timer_list timer;
 478         long size;
 479         struct deferred_flush_table *tables;
 480 };
 481
 482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 483
 484 /* bitmap for indexing intel_iommus */
 485 static int g_num_of_iommus;
 486
 487 static void domain_exit(struct dmar_domain *domain);
 488 static void domain_remove_dev_info(struct dmar_domain *domain);
 489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 490                                      struct device *dev);
 491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 492 static void domain_context_clear(struct intel_iommu *iommu,
 493                                  struct device *dev);
 494 static int domain_detach_iommu(struct dmar_domain *domain,
 495                                struct intel_iommu *iommu);
 496
 497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 498 int dmar_disabled = 0;
 499 #else
 500 int dmar_disabled = 1;
 501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 502
 503 int intel_iommu_enabled = 0;
 504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 505
 506 static int dmar_map_gfx = 1;
 507 static int dmar_forcedac;
 508 static int intel_iommu_strict;
 509 static int intel_iommu_superpage = 1;
 510 static int intel_iommu_ecs = 1;
 511 static int intel_iommu_pasid28;
 512 static int iommu_identity_mapping;
 513
 514 #define IDENTMAP_ALL            1
 515 #define IDENTMAP_GFX            2
 516 #define IDENTMAP_AZALIA         4
 517
 518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 519  * level" translation of DMA requests-without-PASID doesn't actually happen
 520  * unless you also set the NESTE bit in an extended context-entry. Which of
 521  * course means that SVM doesn't work because it's trying to do nested
 522  * translation of the physical addresses it finds in the process page tables,
 523  * through the IOVA->phys mapping found in the "second level" page tables.
 524  *
 525  * The VT-d specification was retroactively changed to change the definition
 526  * of the capability bits and pretend that Broadwell/Skylake never happened...
 527  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 528  * for some reason it was the PASID capability bit which was redefined (from
 529  * bit 28 on BDW/SKL to bit 40 in future).
 530  *
 531  * So our test for ECS needs to eschew those implementations which set the old
 532  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 533  * Unless we are working around the 'pasid28' limitations, that is, by putting
 534  * the device into passthrough mode for normal DMA and thus masking the bug.
 535  */
 536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 537                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 539  * or new capability bits are set. */
 540 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 541                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 542
 543 int intel_iommu_gfx_mapped;
 544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 545
 546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 547 static DEFINE_SPINLOCK(device_domain_lock);
 548 static LIST_HEAD(device_domain_list);
 549
 550 static const struct iommu_ops intel_iommu_ops;
 551
 552 static bool translation_pre_enabled(struct intel_iommu *iommu)
 553 {
 554         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 555 }
 556
 557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 558 {
 559         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 560 }
 561
 562 static void init_translation_status(struct intel_iommu *iommu)
 563 {
 564         u32 gsts;
 565
 566         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 567         if (gsts & DMA_GSTS_TES)
 568                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 569 }
 570
 571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 573 {
 574         return container_of(dom, struct dmar_domain, domain);
 575 }
 576
 577 static int __init intel_iommu_setup(char *str)
 578 {
 579         if (!str)
 580                 return -EINVAL;
 581         while (*str) {
 582                 if (!strncmp(str, "on", 2)) {
 583                         dmar_disabled = 0;
 584                         pr_info("IOMMU enabled\n");
 585                 } else if (!strncmp(str, "off", 3)) {
 586                         dmar_disabled = 1;
 587                         pr_info("IOMMU disabled\n");
 588                 } else if (!strncmp(str, "igfx_off", 8)) {
 589                         dmar_map_gfx = 0;
 590                         pr_info("Disable GFX device mapping\n");
 591                 } else if (!strncmp(str, "forcedac", 8)) {
 592                         pr_info("Forcing DAC for PCI devices\n");
 593                         dmar_forcedac = 1;
 594                 } else if (!strncmp(str, "strict", 6)) {
 595                         pr_info("Disable batched IOTLB flush\n");
 596                         intel_iommu_strict = 1;
 597                 } else if (!strncmp(str, "sp_off", 6)) {
 598                         pr_info("Disable supported super page\n");
 599                         intel_iommu_superpage = 0;
 600                 } else if (!strncmp(str, "ecs_off", 7)) {
 601                         printk(KERN_INFO
 602                                 "Intel-IOMMU: disable extended context table support\n");
 603                         intel_iommu_ecs = 0;
 604                 } else if (!strncmp(str, "pasid28", 7)) {
 605                         printk(KERN_INFO
 606                                 "Intel-IOMMU: enable pre-production PASID support\n");
 607                         intel_iommu_pasid28 = 1;
 608                         iommu_identity_mapping |= IDENTMAP_GFX;
 609                 }
 610
 611                 str += strcspn(str, ",");
 612                 while (*str == ',')
 613                         str++;
 614         }
 615         return 0;
 616 }
 617 __setup("intel_iommu=", intel_iommu_setup);
 618
 619 static struct kmem_cache *iommu_domain_cache;
 620 static struct kmem_cache *iommu_devinfo_cache;
 621
 622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 623 {
 624         struct dmar_domain **domains;
 625         int idx = did >> 8;
 626
 627         domains = iommu->domains[idx];
 628         if (!domains)
 629                 return NULL;
 630
 631         return domains[did & 0xff];
 632 }
 633
 634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 635                              struct dmar_domain *domain)
 636 {
 637         struct dmar_domain **domains;
 638         int idx = did >> 8;
 639
 640         if (!iommu->domains[idx]) {
 641                 size_t size = 256 * sizeof(struct dmar_domain *);
 642                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 643         }
 644
 645         domains = iommu->domains[idx];
 646         if (WARN_ON(!domains))
 647                 return;
 648         else
 649                 domains[did & 0xff] = domain;
 650 }
 651
 652 static inline void *alloc_pgtable_page(int node)
 653 {
 654         struct page *page;
 655         void *vaddr = NULL;
 656
 657         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 658         if (page)
 659                 vaddr = page_address(page);
 660         return vaddr;
 661 }
 662
 663 static inline void free_pgtable_page(void *vaddr)
 664 {
 665         free_page((unsigned long)vaddr);
 666 }
 667
 668 static inline void *alloc_domain_mem(void)
 669 {
 670         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 671 }
 672
 673 static void free_domain_mem(void *vaddr)
 674 {
 675         kmem_cache_free(iommu_domain_cache, vaddr);
 676 }
 677
 678 static inline void * alloc_devinfo_mem(void)
 679 {
 680         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 681 }
 682
 683 static inline void free_devinfo_mem(void *vaddr)
 684 {
 685         kmem_cache_free(iommu_devinfo_cache, vaddr);
 686 }
 687
 688 static inline int domain_type_is_vm(struct dmar_domain *domain)
 689 {
 690         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 691 }
 692
 693 static inline int domain_type_is_si(struct dmar_domain *domain)
 694 {
 695         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 696 }
 697
 698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 699 {
 700         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 701                                 DOMAIN_FLAG_STATIC_IDENTITY);
 702 }
 703
 704 static inline int domain_pfn_supported(struct dmar_domain *domain,
 705                                        unsigned long pfn)
 706 {
 707         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 708
 709         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 710 }
 711
 712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 713 {
 714         unsigned long sagaw;
 715         int agaw = -1;
 716
 717         sagaw = cap_sagaw(iommu->cap);
 718         for (agaw = width_to_agaw(max_gaw);
 719              agaw >= 0; agaw--) {
 720                 if (test_bit(agaw, &sagaw))
 721                         break;
 722         }
 723
 724         return agaw;
 725 }
 726
 727 /*
 728  * Calculate max SAGAW for each iommu.
 729  */
 730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 731 {
 732         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 733 }
 734
 735 /*
 736  * calculate agaw for each iommu.
 737  * "SAGAW" may be different across iommus, use a default agaw, and
 738  * get a supported less agaw for iommus that don't support the default agaw.
 739  */
 740 int iommu_calculate_agaw(struct intel_iommu *iommu)
 741 {
 742         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 743 }
 744
 745 /* This functionin only returns single iommu in a domain */
 746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 747 {
 748         int iommu_id;
 749
 750         /* si_domain and vm domain should not get here. */
 751         BUG_ON(domain_type_is_vm_or_si(domain));
 752         for_each_domain_iommu(iommu_id, domain)
 753                 break;
 754
 755         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 756                 return NULL;
 757
 758         return g_iommus[iommu_id];
 759 }
 760
 761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 762 {
 763         struct dmar_drhd_unit *drhd;
 764         struct intel_iommu *iommu;
 765         bool found = false;
 766         int i;
 767
 768         domain->iommu_coherency = 1;
 769
 770         for_each_domain_iommu(i, domain) {
 771                 found = true;
 772                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 773                         domain->iommu_coherency = 0;
 774                         break;
 775                 }
 776         }
 777         if (found)
 778                 return;
 779
 780         /* No hardware attached; use lowest common denominator */
 781         rcu_read_lock();
 782         for_each_active_iommu(iommu, drhd) {
 783                 if (!ecap_coherent(iommu->ecap)) {
 784                         domain->iommu_coherency = 0;
 785                         break;
 786                 }
 787         }
 788         rcu_read_unlock();
 789 }
 790
 791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 792 {
 793         struct dmar_drhd_unit *drhd;
 794         struct intel_iommu *iommu;
 795         int ret = 1;
 796
 797         rcu_read_lock();
 798         for_each_active_iommu(iommu, drhd) {
 799                 if (iommu != skip) {
 800                         if (!ecap_sc_support(iommu->ecap)) {
 801                                 ret = 0;
 802                                 break;
 803                         }
 804                 }
 805         }
 806         rcu_read_unlock();
 807
 808         return ret;
 809 }
 810
 811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 812 {
 813         struct dmar_drhd_unit *drhd;
 814         struct intel_iommu *iommu;
 815         int mask = 0xf;
 816
 817         if (!intel_iommu_superpage) {
 818                 return 0;
 819         }
 820
 821         /* set iommu_superpage to the smallest common denominator */
 822         rcu_read_lock();
 823         for_each_active_iommu(iommu, drhd) {
 824                 if (iommu != skip) {
 825                         mask &= cap_super_page_val(iommu->cap);
 826                         if (!mask)
 827                                 break;
 828                 }
 829         }
 830         rcu_read_unlock();
 831
 832         return fls(mask);
 833 }
 834
 835 /* Some capabilities may be different across iommus */
 836 static void domain_update_iommu_cap(struct dmar_domain *domain)
 837 {
 838         domain_update_iommu_coherency(domain);
 839         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 840         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 841 }
 842
 843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 844                                                        u8 bus, u8 devfn, int alloc)
 845 {
 846         struct root_entry *root = &iommu->root_entry[bus];
 847         struct context_entry *context;
 848         u64 *entry;
 849
 850         entry = &root->lo;
 851         if (ecs_enabled(iommu)) {
 852                 if (devfn >= 0x80) {
 853                         devfn -= 0x80;
 854                         entry = &root->hi;
 855                 }
 856                 devfn *= 2;
 857         }
 858         if (*entry & 1)
 859                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 860         else {
 861                 unsigned long phy_addr;
 862                 if (!alloc)
 863                         return NULL;
 864
 865                 context = alloc_pgtable_page(iommu->node);
 866                 if (!context)
 867                         return NULL;
 868
 869                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 870                 phy_addr = virt_to_phys((void *)context);
 871                 *entry = phy_addr | 1;
 872                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 873         }
 874         return &context[devfn];
 875 }
 876
 877 static int iommu_dummy(struct device *dev)
 878 {
 879         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 880 }
 881
 882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 883 {
 884         struct dmar_drhd_unit *drhd = NULL;
 885         struct intel_iommu *iommu;
 886         struct device *tmp;
 887         struct pci_dev *ptmp, *pdev = NULL;
 888         u16 segment = 0;
 889         int i;
 890
 891         if (iommu_dummy(dev))
 892                 return NULL;
 893
 894         if (dev_is_pci(dev)) {
 895                 pdev = to_pci_dev(dev);
 896                 segment = pci_domain_nr(pdev->bus);
 897         } else if (has_acpi_companion(dev))
 898                 dev = &ACPI_COMPANION(dev)->dev;
 899
 900         rcu_read_lock();
 901         for_each_active_iommu(iommu, drhd) {
 902                 if (pdev && segment != drhd->segment)
 903                         continue;
 904
 905                 for_each_active_dev_scope(drhd->devices,
 906                                           drhd->devices_cnt, i, tmp) {
 907                         if (tmp == dev) {
 908                                 *bus = drhd->devices[i].bus;
 909                                 *devfn = drhd->devices[i].devfn;
 910                                 goto out;
 911                         }
 912
 913                         if (!pdev || !dev_is_pci(tmp))
 914                                 continue;
 915
 916                         ptmp = to_pci_dev(tmp);
 917                         if (ptmp->subordinate &&
 918                             ptmp->subordinate->number <= pdev->bus->number &&
 919                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 920                                 goto got_pdev;
 921                 }
 922
 923                 if (pdev && drhd->include_all) {
 924                 got_pdev:
 925                         *bus = pdev->bus->number;
 926                         *devfn = pdev->devfn;
 927                         goto out;
 928                 }
 929         }
 930         iommu = NULL;
 931  out:
 932         rcu_read_unlock();
 933
 934         return iommu;
 935 }
 936
 937 static void domain_flush_cache(struct dmar_domain *domain,
 938                                void *addr, int size)
 939 {
 940         if (!domain->iommu_coherency)
 941                 clflush_cache_range(addr, size);
 942 }
 943
 944 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 945 {
 946         struct context_entry *context;
 947         int ret = 0;
 948         unsigned long flags;
 949
 950         spin_lock_irqsave(&iommu->lock, flags);
 951         context = iommu_context_addr(iommu, bus, devfn, 0);
 952         if (context)
 953                 ret = context_present(context);
 954         spin_unlock_irqrestore(&iommu->lock, flags);
 955         return ret;
 956 }
 957
 958 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 959 {
 960         struct context_entry *context;
 961         unsigned long flags;
 962
 963         spin_lock_irqsave(&iommu->lock, flags);
 964         context = iommu_context_addr(iommu, bus, devfn, 0);
 965         if (context) {
 966                 context_clear_entry(context);
 967                 __iommu_flush_cache(iommu, context, sizeof(*context));
 968         }
 969         spin_unlock_irqrestore(&iommu->lock, flags);
 970 }
 971
 972 static void free_context_table(struct intel_iommu *iommu)
 973 {
 974         int i;
 975         unsigned long flags;
 976         struct context_entry *context;
 977
 978         spin_lock_irqsave(&iommu->lock, flags);
 979         if (!iommu->root_entry) {
 980                 goto out;
 981         }
 982         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 983                 context = iommu_context_addr(iommu, i, 0, 0);
 984                 if (context)
 985                         free_pgtable_page(context);
 986
 987                 if (!ecs_enabled(iommu))
 988                         continue;
 989
 990                 context = iommu_context_addr(iommu, i, 0x80, 0);
 991                 if (context)
 992                         free_pgtable_page(context);
 993
 994         }
 995         free_pgtable_page(iommu->root_entry);
 996         iommu->root_entry = NULL;
 997 out:
 998         spin_unlock_irqrestore(&iommu->lock, flags);
 999 }
1000
1001 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1002                                       unsigned long pfn, int *target_level)
1003 {
1004         struct dma_pte *parent, *pte = NULL;
1005         int level = agaw_to_level(domain->agaw);
1006         int offset;
1007
1008         BUG_ON(!domain->pgd);
1009
1010         if (!domain_pfn_supported(domain, pfn))
1011                 /* Address beyond IOMMU's addressing capabilities. */
1012                 return NULL;
1013
1014         parent = domain->pgd;
1015
1016         while (1) {
1017                 void *tmp_page;
1018
1019                 offset = pfn_level_offset(pfn, level);
1020                 pte = &parent[offset];
1021                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1022                         break;
1023                 if (level == *target_level)
1024                         break;
1025
1026                 if (!dma_pte_present(pte)) {
1027                         uint64_t pteval;
1028
1029                         tmp_page = alloc_pgtable_page(domain->nid);
1030
1031                         if (!tmp_page)
1032                                 return NULL;
1033
1034                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1035                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1036                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1037                                 /* Someone else set it while we were thinking; use theirs. */
1038                                 free_pgtable_page(tmp_page);
1039                         else
1040                                 domain_flush_cache(domain, pte, sizeof(*pte));
1041                 }
1042                 if (level == 1)
1043                         break;
1044
1045                 parent = phys_to_virt(dma_pte_addr(pte));
1046                 level--;
1047         }
1048
1049         if (!*target_level)
1050                 *target_level = level;
1051
1052         return pte;
1053 }
1054
1055
1056 /* return address's pte at specific level */
1057 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1058                                          unsigned long pfn,
1059                                          int level, int *large_page)
1060 {
1061         struct dma_pte *parent, *pte = NULL;
1062         int total = agaw_to_level(domain->agaw);
1063         int offset;
1064
1065         parent = domain->pgd;
1066         while (level <= total) {
1067                 offset = pfn_level_offset(pfn, total);
1068                 pte = &parent[offset];
1069                 if (level == total)
1070                         return pte;
1071
1072                 if (!dma_pte_present(pte)) {
1073                         *large_page = total;
1074                         break;
1075                 }
1076
1077                 if (dma_pte_superpage(pte)) {
1078                         *large_page = total;
1079                         return pte;
1080                 }
1081
1082                 parent = phys_to_virt(dma_pte_addr(pte));
1083                 total--;
1084         }
1085         return NULL;
1086 }
1087
1088 /* clear last level pte, a tlb flush should be followed */
1089 static void dma_pte_clear_range(struct dmar_domain *domain,
1090                                 unsigned long start_pfn,
1091                                 unsigned long last_pfn)
1092 {
1093         unsigned int large_page = 1;
1094         struct dma_pte *first_pte, *pte;
1095
1096         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1098         BUG_ON(start_pfn > last_pfn);
1099
1100         /* we don't need lock here; nobody else touches the iova range */
1101         do {
1102                 large_page = 1;
1103                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1104                 if (!pte) {
1105                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1106                         continue;
1107                 }
1108                 do {
1109                         dma_clear_pte(pte);
1110                         start_pfn += lvl_to_nr_pages(large_page);
1111                         pte++;
1112                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1113
1114                 domain_flush_cache(domain, first_pte,
1115                                    (void *)pte - (void *)first_pte);
1116
1117         } while (start_pfn && start_pfn <= last_pfn);
1118 }
1119
1120 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1121                                struct dma_pte *pte, unsigned long pfn,
1122                                unsigned long start_pfn, unsigned long last_pfn)
1123 {
1124         pfn = max(start_pfn, pfn);
1125         pte = &pte[pfn_level_offset(pfn, level)];
1126
1127         do {
1128                 unsigned long level_pfn;
1129                 struct dma_pte *level_pte;
1130
1131                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1132                         goto next;
1133
1134                 level_pfn = pfn & level_mask(level - 1);
1135                 level_pte = phys_to_virt(dma_pte_addr(pte));
1136
1137                 if (level > 2)
1138                         dma_pte_free_level(domain, level - 1, level_pte,
1139                                            level_pfn, start_pfn, last_pfn);
1140
1141                 /* If range covers entire pagetable, free it */
1142                 if (!(start_pfn > level_pfn ||
1143                       last_pfn < level_pfn + level_size(level) - 1)) {
1144                         dma_clear_pte(pte);
1145                         domain_flush_cache(domain, pte, sizeof(*pte));
1146                         free_pgtable_page(level_pte);
1147                 }
1148 next:
1149                 pfn += level_size(level);
1150         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1151 }
1152
1153 /* clear last level (leaf) ptes and free page table pages. */
1154 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1155                                    unsigned long start_pfn,
1156                                    unsigned long last_pfn)
1157 {
1158         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1159         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1160         BUG_ON(start_pfn > last_pfn);
1161
1162         dma_pte_clear_range(domain, start_pfn, last_pfn);
1163
1164         /* We don't need lock here; nobody else touches the iova range */
1165         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1166                            domain->pgd, 0, start_pfn, last_pfn);
1167
1168         /* free pgd */
1169         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1170                 free_pgtable_page(domain->pgd);
1171                 domain->pgd = NULL;
1172         }
1173 }
1174
1175 /* When a page at a given level is being unlinked from its parent, we don't
1176    need to *modify* it at all. All we need to do is make a list of all the
1177    pages which can be freed just as soon as we've flushed the IOTLB and we
1178    know the hardware page-walk will no longer touch them.
1179    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1180    be freed. */
1181 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1182                                             int level, struct dma_pte *pte,
1183                                             struct page *freelist)
1184 {
1185         struct page *pg;
1186
1187         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1188         pg->freelist = freelist;
1189         freelist = pg;
1190
1191         if (level == 1)
1192                 return freelist;
1193
1194         pte = page_address(pg);
1195         do {
1196                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1197                         freelist = dma_pte_list_pagetables(domain, level - 1,
1198                                                            pte, freelist);
1199                 pte++;
1200         } while (!first_pte_in_page(pte));
1201
1202         return freelist;
1203 }
1204
1205 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1206                                         struct dma_pte *pte, unsigned long pfn,
1207                                         unsigned long start_pfn,
1208                                         unsigned long last_pfn,
1209                                         struct page *freelist)
1210 {
1211         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1212
1213         pfn = max(start_pfn, pfn);
1214         pte = &pte[pfn_level_offset(pfn, level)];
1215
1216         do {
1217                 unsigned long level_pfn;
1218
1219                 if (!dma_pte_present(pte))
1220                         goto next;
1221
1222                 level_pfn = pfn & level_mask(level);
1223
1224                 /* If range covers entire pagetable, free it */
1225                 if (start_pfn <= level_pfn &&
1226                     last_pfn >= level_pfn + level_size(level) - 1) {
1227                         /* These suborbinate page tables are going away entirely. Don't
1228                            bother to clear them; we're just going to *free* them. */
1229                         if (level > 1 && !dma_pte_superpage(pte))
1230                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1231
1232                         dma_clear_pte(pte);
1233                         if (!first_pte)
1234                                 first_pte = pte;
1235                         last_pte = pte;
1236                 } else if (level > 1) {
1237                         /* Recurse down into a level that isn't *entirely* obsolete */
1238                         freelist = dma_pte_clear_level(domain, level - 1,
1239                                                        phys_to_virt(dma_pte_addr(pte)),
1240                                                        level_pfn, start_pfn, last_pfn,
1241                                                        freelist);
1242                 }
1243 next:
1244                 pfn += level_size(level);
1245         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1246
1247         if (first_pte)
1248                 domain_flush_cache(domain, first_pte,
1249                                    (void *)++last_pte - (void *)first_pte);
1250
1251         return freelist;
1252 }
1253
1254 /* We can't just free the pages because the IOMMU may still be walking
1255    the page tables, and may have cached the intermediate levels. The
1256    pages can only be freed after the IOTLB flush has been done. */
1257 static struct page *domain_unmap(struct dmar_domain *domain,
1258                                  unsigned long start_pfn,
1259                                  unsigned long last_pfn)
1260 {
1261         struct page *freelist = NULL;
1262
1263         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1264         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1265         BUG_ON(start_pfn > last_pfn);
1266
1267         /* we don't need lock here; nobody else touches the iova range */
1268         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1269                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1270
1271         /* free pgd */
1272         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1273                 struct page *pgd_page = virt_to_page(domain->pgd);
1274                 pgd_page->freelist = freelist;
1275                 freelist = pgd_page;
1276
1277                 domain->pgd = NULL;
1278         }
1279
1280         return freelist;
1281 }
1282
1283 static void dma_free_pagelist(struct page *freelist)
1284 {
1285         struct page *pg;
1286
1287         while ((pg = freelist)) {
1288                 freelist = pg->freelist;
1289                 free_pgtable_page(page_address(pg));
1290         }
1291 }
1292
1293 /* iommu handling */
1294 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1295 {
1296         struct root_entry *root;
1297         unsigned long flags;
1298
1299         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1300         if (!root) {
1301                 pr_err("Allocating root entry for %s failed\n",
1302                         iommu->name);
1303                 return -ENOMEM;
1304         }
1305
1306         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1307
1308         spin_lock_irqsave(&iommu->lock, flags);
1309         iommu->root_entry = root;
1310         spin_unlock_irqrestore(&iommu->lock, flags);
1311
1312         return 0;
1313 }
1314
1315 static void iommu_set_root_entry(struct intel_iommu *iommu)
1316 {
1317         u64 addr;
1318         u32 sts;
1319         unsigned long flag;
1320
1321         addr = virt_to_phys(iommu->root_entry);
1322         if (ecs_enabled(iommu))
1323                 addr |= DMA_RTADDR_RTT;
1324
1325         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1326         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1327
1328         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1329
1330         /* Make sure hardware complete it */
1331         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1332                       readl, (sts & DMA_GSTS_RTPS), sts);
1333
1334         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335 }
1336
1337 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1338 {
1339         u32 val;
1340         unsigned long flag;
1341
1342         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1343                 return;
1344
1345         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1346         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1347
1348         /* Make sure hardware complete it */
1349         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1350                       readl, (!(val & DMA_GSTS_WBFS)), val);
1351
1352         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1353 }
1354
1355 /* return value determine if we need a write buffer flush */
1356 static void __iommu_flush_context(struct intel_iommu *iommu,
1357                                   u16 did, u16 source_id, u8 function_mask,
1358                                   u64 type)
1359 {
1360         u64 val = 0;
1361         unsigned long flag;
1362
1363         switch (type) {
1364         case DMA_CCMD_GLOBAL_INVL:
1365                 val = DMA_CCMD_GLOBAL_INVL;
1366                 break;
1367         case DMA_CCMD_DOMAIN_INVL:
1368                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1369                 break;
1370         case DMA_CCMD_DEVICE_INVL:
1371                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1372                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1373                 break;
1374         default:
1375                 BUG();
1376         }
1377         val |= DMA_CCMD_ICC;
1378
1379         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1380         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1381
1382         /* Make sure hardware complete it */
1383         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1384                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1385
1386         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1387 }
1388
1389 /* return value determine if we need a write buffer flush */
1390 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1391                                 u64 addr, unsigned int size_order, u64 type)
1392 {
1393         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1394         u64 val = 0, val_iva = 0;
1395         unsigned long flag;
1396
1397         switch (type) {
1398         case DMA_TLB_GLOBAL_FLUSH:
1399                 /* global flush doesn't need set IVA_REG */
1400                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1401                 break;
1402         case DMA_TLB_DSI_FLUSH:
1403                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1404                 break;
1405         case DMA_TLB_PSI_FLUSH:
1406                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1407                 /* IH bit is passed in as part of address */
1408                 val_iva = size_order | addr;
1409                 break;
1410         default:
1411                 BUG();
1412         }
1413         /* Note: set drain read/write */
1414 #if 0
1415         /*
1416          * This is probably to be super secure.. Looks like we can
1417          * ignore it without any impact.
1418          */
1419         if (cap_read_drain(iommu->cap))
1420                 val |= DMA_TLB_READ_DRAIN;
1421 #endif
1422         if (cap_write_drain(iommu->cap))
1423                 val |= DMA_TLB_WRITE_DRAIN;
1424
1425         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1426         /* Note: Only uses first TLB reg currently */
1427         if (val_iva)
1428                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1429         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1430
1431         /* Make sure hardware complete it */
1432         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1433                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1434
1435         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1436
1437         /* check IOTLB invalidation granularity */
1438         if (DMA_TLB_IAIG(val) == 0)
1439                 pr_err("Flush IOTLB failed\n");
1440         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1441                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1442                         (unsigned long long)DMA_TLB_IIRG(type),
1443                         (unsigned long long)DMA_TLB_IAIG(val));
1444 }
1445
1446 static struct device_domain_info *
1447 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1448                          u8 bus, u8 devfn)
1449 {
1450         struct device_domain_info *info;
1451
1452         assert_spin_locked(&device_domain_lock);
1453
1454         if (!iommu->qi)
1455                 return NULL;
1456
1457         list_for_each_entry(info, &domain->devices, link)
1458                 if (info->iommu == iommu && info->bus == bus &&
1459                     info->devfn == devfn) {
1460                         if (info->ats_supported && info->dev)
1461                                 return info;
1462                         break;
1463                 }
1464
1465         return NULL;
1466 }
1467
1468 static void domain_update_iotlb(struct dmar_domain *domain)
1469 {
1470         struct device_domain_info *info;
1471         bool has_iotlb_device = false;
1472
1473         assert_spin_locked(&device_domain_lock);
1474
1475         list_for_each_entry(info, &domain->devices, link) {
1476                 struct pci_dev *pdev;
1477
1478                 if (!info->dev || !dev_is_pci(info->dev))
1479                         continue;
1480
1481                 pdev = to_pci_dev(info->dev);
1482                 if (pdev->ats_enabled) {
1483                         has_iotlb_device = true;
1484                         break;
1485                 }
1486         }
1487
1488         domain->has_iotlb_device = has_iotlb_device;
1489 }
1490
1491 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1492 {
1493         struct pci_dev *pdev;
1494
1495         assert_spin_locked(&device_domain_lock);
1496
1497         if (!info || !dev_is_pci(info->dev))
1498                 return;
1499
1500         pdev = to_pci_dev(info->dev);
1501
1502 #ifdef CONFIG_INTEL_IOMMU_SVM
1503         /* The PCIe spec, in its wisdom, declares that the behaviour of
1504            the device if you enable PASID support after ATS support is
1505            undefined. So always enable PASID support on devices which
1506            have it, even if we can't yet know if we're ever going to
1507            use it. */
1508         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1509                 info->pasid_enabled = 1;
1510
1511         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1512                 info->pri_enabled = 1;
1513 #endif
1514         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1515                 info->ats_enabled = 1;
1516                 domain_update_iotlb(info->domain);
1517                 info->ats_qdep = pci_ats_queue_depth(pdev);
1518         }
1519 }
1520
1521 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1522 {
1523         struct pci_dev *pdev;
1524
1525         assert_spin_locked(&device_domain_lock);
1526
1527         if (!dev_is_pci(info->dev))
1528                 return;
1529
1530         pdev = to_pci_dev(info->dev);
1531
1532         if (info->ats_enabled) {
1533                 pci_disable_ats(pdev);
1534                 info->ats_enabled = 0;
1535                 domain_update_iotlb(info->domain);
1536         }
1537 #ifdef CONFIG_INTEL_IOMMU_SVM
1538         if (info->pri_enabled) {
1539                 pci_disable_pri(pdev);
1540                 info->pri_enabled = 0;
1541         }
1542         if (info->pasid_enabled) {
1543                 pci_disable_pasid(pdev);
1544                 info->pasid_enabled = 0;
1545         }
1546 #endif
1547 }
1548
1549 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1550                                   u64 addr, unsigned mask)
1551 {
1552         u16 sid, qdep;
1553         unsigned long flags;
1554         struct device_domain_info *info;
1555
1556         if (!domain->has_iotlb_device)
1557                 return;
1558
1559         spin_lock_irqsave(&device_domain_lock, flags);
1560         list_for_each_entry(info, &domain->devices, link) {
1561                 if (!info->ats_enabled)
1562                         continue;
1563
1564                 sid = info->bus << 8 | info->devfn;
1565                 qdep = info->ats_qdep;
1566                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1567         }
1568         spin_unlock_irqrestore(&device_domain_lock, flags);
1569 }
1570
1571 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1572                                   struct dmar_domain *domain,
1573                                   unsigned long pfn, unsigned int pages,
1574                                   int ih, int map)
1575 {
1576         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1577         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1578         u16 did = domain->iommu_did[iommu->seq_id];
1579
1580         BUG_ON(pages == 0);
1581
1582         if (ih)
1583                 ih = 1 << 6;
1584         /*
1585          * Fallback to domain selective flush if no PSI support or the size is
1586          * too big.
1587          * PSI requires page size to be 2 ^ x, and the base address is naturally
1588          * aligned to the size
1589          */
1590         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1591                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1592                                                 DMA_TLB_DSI_FLUSH);
1593         else
1594                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1595                                                 DMA_TLB_PSI_FLUSH);
1596
1597         /*
1598          * In caching mode, changes of pages from non-present to present require
1599          * flush. However, device IOTLB doesn't need to be flushed in this case.
1600          */
1601         if (!cap_caching_mode(iommu->cap) || !map)
1602                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1603                                       addr, mask);
1604 }
1605
1606 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1607 {
1608         u32 pmen;
1609         unsigned long flags;
1610
1611         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1612         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1613         pmen &= ~DMA_PMEN_EPM;
1614         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1615
1616         /* wait for the protected region status bit to clear */
1617         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1618                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1619
1620         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1621 }
1622
1623 static void iommu_enable_translation(struct intel_iommu *iommu)
1624 {
1625         u32 sts;
1626         unsigned long flags;
1627
1628         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1629         iommu->gcmd |= DMA_GCMD_TE;
1630         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1631
1632         /* Make sure hardware complete it */
1633         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1634                       readl, (sts & DMA_GSTS_TES), sts);
1635
1636         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637 }
1638
1639 static void iommu_disable_translation(struct intel_iommu *iommu)
1640 {
1641         u32 sts;
1642         unsigned long flag;
1643
1644         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1645         iommu->gcmd &= ~DMA_GCMD_TE;
1646         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648         /* Make sure hardware complete it */
1649         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1650                       readl, (!(sts & DMA_GSTS_TES)), sts);
1651
1652         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1653 }
1654
1655
1656 static int iommu_init_domains(struct intel_iommu *iommu)
1657 {
1658         u32 ndomains, nlongs;
1659         size_t size;
1660
1661         ndomains = cap_ndoms(iommu->cap);
1662         pr_debug("%s: Number of Domains supported <%d>\n",
1663                  iommu->name, ndomains);
1664         nlongs = BITS_TO_LONGS(ndomains);
1665
1666         spin_lock_init(&iommu->lock);
1667
1668         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1669         if (!iommu->domain_ids) {
1670                 pr_err("%s: Allocating domain id array failed\n",
1671                        iommu->name);
1672                 return -ENOMEM;
1673         }
1674
1675         size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1676         iommu->domains = kzalloc(size, GFP_KERNEL);
1677
1678         if (iommu->domains) {
1679                 size = 256 * sizeof(struct dmar_domain *);
1680                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1681         }
1682
1683         if (!iommu->domains || !iommu->domains[0]) {
1684                 pr_err("%s: Allocating domain array failed\n",
1685                        iommu->name);
1686                 kfree(iommu->domain_ids);
1687                 kfree(iommu->domains);
1688                 iommu->domain_ids = NULL;
1689                 iommu->domains    = NULL;
1690                 return -ENOMEM;
1691         }
1692
1693
1694
1695         /*
1696          * If Caching mode is set, then invalid translations are tagged
1697          * with domain-id 0, hence we need to pre-allocate it. We also
1698          * use domain-id 0 as a marker for non-allocated domain-id, so
1699          * make sure it is not used for a real domain.
1700          */
1701         set_bit(0, iommu->domain_ids);
1702
1703         return 0;
1704 }
1705
1706 static void disable_dmar_iommu(struct intel_iommu *iommu)
1707 {
1708         struct device_domain_info *info, *tmp;
1709         unsigned long flags;
1710
1711         if (!iommu->domains || !iommu->domain_ids)
1712                 return;
1713
1714         spin_lock_irqsave(&device_domain_lock, flags);
1715         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1716                 struct dmar_domain *domain;
1717
1718                 if (info->iommu != iommu)
1719                         continue;
1720
1721                 if (!info->dev || !info->domain)
1722                         continue;
1723
1724                 domain = info->domain;
1725
1726                 dmar_remove_one_dev_info(domain, info->dev);
1727
1728                 if (!domain_type_is_vm_or_si(domain))
1729                         domain_exit(domain);
1730         }
1731         spin_unlock_irqrestore(&device_domain_lock, flags);
1732
1733         if (iommu->gcmd & DMA_GCMD_TE)
1734                 iommu_disable_translation(iommu);
1735 }
1736
1737 static void free_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739         if ((iommu->domains) && (iommu->domain_ids)) {
1740                 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1741                 int i;
1742
1743                 for (i = 0; i < elems; i++)
1744                         kfree(iommu->domains[i]);
1745                 kfree(iommu->domains);
1746                 kfree(iommu->domain_ids);
1747                 iommu->domains = NULL;
1748                 iommu->domain_ids = NULL;
1749         }
1750
1751         g_iommus[iommu->seq_id] = NULL;
1752
1753         /* free context mapping */
1754         free_context_table(iommu);
1755
1756 #ifdef CONFIG_INTEL_IOMMU_SVM
1757         if (pasid_enabled(iommu)) {
1758                 if (ecap_prs(iommu->ecap))
1759                         intel_svm_finish_prq(iommu);
1760                 intel_svm_free_pasid_tables(iommu);
1761         }
1762 #endif
1763 }
1764
1765 static struct dmar_domain *alloc_domain(int flags)
1766 {
1767         struct dmar_domain *domain;
1768
1769         domain = alloc_domain_mem();
1770         if (!domain)
1771                 return NULL;
1772
1773         memset(domain, 0, sizeof(*domain));
1774         domain->nid = -1;
1775         domain->flags = flags;
1776         domain->has_iotlb_device = false;
1777         INIT_LIST_HEAD(&domain->devices);
1778
1779         return domain;
1780 }
1781
1782 /* Must be called with iommu->lock */
1783 static int domain_attach_iommu(struct dmar_domain *domain,
1784                                struct intel_iommu *iommu)
1785 {
1786         unsigned long ndomains;
1787         int num;
1788
1789         assert_spin_locked(&device_domain_lock);
1790         assert_spin_locked(&iommu->lock);
1791
1792         domain->iommu_refcnt[iommu->seq_id] += 1;
1793         domain->iommu_count += 1;
1794         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1795                 ndomains = cap_ndoms(iommu->cap);
1796                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1797
1798                 if (num >= ndomains) {
1799                         pr_err("%s: No free domain ids\n", iommu->name);
1800                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1801                         domain->iommu_count -= 1;
1802                         return -ENOSPC;
1803                 }
1804
1805                 set_bit(num, iommu->domain_ids);
1806                 set_iommu_domain(iommu, num, domain);
1807
1808                 domain->iommu_did[iommu->seq_id] = num;
1809                 domain->nid                      = iommu->node;
1810
1811                 domain_update_iommu_cap(domain);
1812         }
1813
1814         return 0;
1815 }
1816
1817 static int domain_detach_iommu(struct dmar_domain *domain,
1818                                struct intel_iommu *iommu)
1819 {
1820         int num, count = INT_MAX;
1821
1822         assert_spin_locked(&device_domain_lock);
1823         assert_spin_locked(&iommu->lock);
1824
1825         domain->iommu_refcnt[iommu->seq_id] -= 1;
1826         count = --domain->iommu_count;
1827         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1828                 num = domain->iommu_did[iommu->seq_id];
1829                 clear_bit(num, iommu->domain_ids);
1830                 set_iommu_domain(iommu, num, NULL);
1831
1832                 domain_update_iommu_cap(domain);
1833                 domain->iommu_did[iommu->seq_id] = 0;
1834         }
1835
1836         return count;
1837 }
1838
1839 static struct iova_domain reserved_iova_list;
1840 static struct lock_class_key reserved_rbtree_key;
1841
1842 static int dmar_init_reserved_ranges(void)
1843 {
1844         struct pci_dev *pdev = NULL;
1845         struct iova *iova;
1846         int i;
1847
1848         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1849                         DMA_32BIT_PFN);
1850
1851         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852                 &reserved_rbtree_key);
1853
1854         /* IOAPIC ranges shouldn't be accessed by DMA */
1855         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856                 IOVA_PFN(IOAPIC_RANGE_END));
1857         if (!iova) {
1858                 pr_err("Reserve IOAPIC range failed\n");
1859                 return -ENODEV;
1860         }
1861
1862         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863         for_each_pci_dev(pdev) {
1864                 struct resource *r;
1865
1866                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867                         r = &pdev->resource[i];
1868                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869                                 continue;
1870                         iova = reserve_iova(&reserved_iova_list,
1871                                             IOVA_PFN(r->start),
1872                                             IOVA_PFN(r->end));
1873                         if (!iova) {
1874                                 pr_err("Reserve iova failed\n");
1875                                 return -ENODEV;
1876                         }
1877                 }
1878         }
1879         return 0;
1880 }
1881
1882 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883 {
1884         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885 }
1886
1887 static inline int guestwidth_to_adjustwidth(int gaw)
1888 {
1889         int agaw;
1890         int r = (gaw - 12) % 9;
1891
1892         if (r == 0)
1893                 agaw = gaw;
1894         else
1895                 agaw = gaw + 9 - r;
1896         if (agaw > 64)
1897                 agaw = 64;
1898         return agaw;
1899 }
1900
1901 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902                        int guest_width)
1903 {
1904         int adjust_width, agaw;
1905         unsigned long sagaw;
1906
1907         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1908                         DMA_32BIT_PFN);
1909         domain_reserve_special_ranges(domain);
1910
1911         /* calculate AGAW */
1912         if (guest_width > cap_mgaw(iommu->cap))
1913                 guest_width = cap_mgaw(iommu->cap);
1914         domain->gaw = guest_width;
1915         adjust_width = guestwidth_to_adjustwidth(guest_width);
1916         agaw = width_to_agaw(adjust_width);
1917         sagaw = cap_sagaw(iommu->cap);
1918         if (!test_bit(agaw, &sagaw)) {
1919                 /* hardware doesn't support it, choose a bigger one */
1920                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1921                 agaw = find_next_bit(&sagaw, 5, agaw);
1922                 if (agaw >= 5)
1923                         return -ENODEV;
1924         }
1925         domain->agaw = agaw;
1926
1927         if (ecap_coherent(iommu->ecap))
1928                 domain->iommu_coherency = 1;
1929         else
1930                 domain->iommu_coherency = 0;
1931
1932         if (ecap_sc_support(iommu->ecap))
1933                 domain->iommu_snooping = 1;
1934         else
1935                 domain->iommu_snooping = 0;
1936
1937         if (intel_iommu_superpage)
1938                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1939         else
1940                 domain->iommu_superpage = 0;
1941
1942         domain->nid = iommu->node;
1943
1944         /* always allocate the top pgd */
1945         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1946         if (!domain->pgd)
1947                 return -ENOMEM;
1948         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1949         return 0;
1950 }
1951
1952 static void domain_exit(struct dmar_domain *domain)
1953 {
1954         struct page *freelist = NULL;
1955
1956         /* Domain 0 is reserved, so dont process it */
1957         if (!domain)
1958                 return;
1959
1960         /* Flush any lazy unmaps that may reference this domain */
1961         if (!intel_iommu_strict) {
1962                 int cpu;
1963
1964                 for_each_possible_cpu(cpu)
1965                         flush_unmaps_timeout(cpu);
1966         }
1967
1968         /* Remove associated devices and clear attached or cached domains */
1969         rcu_read_lock();
1970         domain_remove_dev_info(domain);
1971         rcu_read_unlock();
1972
1973         /* destroy iovas */
1974         put_iova_domain(&domain->iovad);
1975
1976         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1977
1978         dma_free_pagelist(freelist);
1979
1980         free_domain_mem(domain);
1981 }
1982
1983 static int domain_context_mapping_one(struct dmar_domain *domain,
1984                                       struct intel_iommu *iommu,
1985                                       u8 bus, u8 devfn)
1986 {
1987         u16 did = domain->iommu_did[iommu->seq_id];
1988         int translation = CONTEXT_TT_MULTI_LEVEL;
1989         struct device_domain_info *info = NULL;
1990         struct context_entry *context;
1991         unsigned long flags;
1992         struct dma_pte *pgd;
1993         int ret, agaw;
1994
1995         WARN_ON(did == 0);
1996
1997         if (hw_pass_through && domain_type_is_si(domain))
1998                 translation = CONTEXT_TT_PASS_THROUGH;
1999
2000         pr_debug("Set context mapping for %02x:%02x.%d\n",
2001                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2002
2003         BUG_ON(!domain->pgd);
2004
2005         spin_lock_irqsave(&device_domain_lock, flags);
2006         spin_lock(&iommu->lock);
2007
2008         ret = -ENOMEM;
2009         context = iommu_context_addr(iommu, bus, devfn, 1);
2010         if (!context)
2011                 goto out_unlock;
2012
2013         ret = 0;
2014         if (context_present(context))
2015                 goto out_unlock;
2016
2017         pgd = domain->pgd;
2018
2019         context_clear_entry(context);
2020         context_set_domain_id(context, did);
2021
2022         /*
2023          * Skip top levels of page tables for iommu which has less agaw
2024          * than default.  Unnecessary for PT mode.
2025          */
2026         if (translation != CONTEXT_TT_PASS_THROUGH) {
2027                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2028                         ret = -ENOMEM;
2029                         pgd = phys_to_virt(dma_pte_addr(pgd));
2030                         if (!dma_pte_present(pgd))
2031                                 goto out_unlock;
2032                 }
2033
2034                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2035                 if (info && info->ats_supported)
2036                         translation = CONTEXT_TT_DEV_IOTLB;
2037                 else
2038                         translation = CONTEXT_TT_MULTI_LEVEL;
2039
2040                 context_set_address_root(context, virt_to_phys(pgd));
2041                 context_set_address_width(context, iommu->agaw);
2042         } else {
2043                 /*
2044                  * In pass through mode, AW must be programmed to
2045                  * indicate the largest AGAW value supported by
2046                  * hardware. And ASR is ignored by hardware.
2047                  */
2048                 context_set_address_width(context, iommu->msagaw);
2049         }
2050
2051         context_set_translation_type(context, translation);
2052         context_set_fault_enable(context);
2053         context_set_present(context);
2054         domain_flush_cache(domain, context, sizeof(*context));
2055
2056         /*
2057          * It's a non-present to present mapping. If hardware doesn't cache
2058          * non-present entry we only need to flush the write-buffer. If the
2059          * _does_ cache non-present entries, then it does so in the special
2060          * domain #0, which we have to flush:
2061          */
2062         if (cap_caching_mode(iommu->cap)) {
2063                 iommu->flush.flush_context(iommu, 0,
2064                                            (((u16)bus) << 8) | devfn,
2065                                            DMA_CCMD_MASK_NOBIT,
2066                                            DMA_CCMD_DEVICE_INVL);
2067                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2068         } else {
2069                 iommu_flush_write_buffer(iommu);
2070         }
2071         iommu_enable_dev_iotlb(info);
2072
2073         ret = 0;
2074
2075 out_unlock:
2076         spin_unlock(&iommu->lock);
2077         spin_unlock_irqrestore(&device_domain_lock, flags);
2078
2079         return 0;
2080 }
2081
2082 struct domain_context_mapping_data {
2083         struct dmar_domain *domain;
2084         struct intel_iommu *iommu;
2085 };
2086
2087 static int domain_context_mapping_cb(struct pci_dev *pdev,
2088                                      u16 alias, void *opaque)
2089 {
2090         struct domain_context_mapping_data *data = opaque;
2091
2092         return domain_context_mapping_one(data->domain, data->iommu,
2093                                           PCI_BUS_NUM(alias), alias & 0xff);
2094 }
2095
2096 static int
2097 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2098 {
2099         struct intel_iommu *iommu;
2100         u8 bus, devfn;
2101         struct domain_context_mapping_data data;
2102
2103         iommu = device_to_iommu(dev, &bus, &devfn);
2104         if (!iommu)
2105                 return -ENODEV;
2106
2107         if (!dev_is_pci(dev))
2108                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2109
2110         data.domain = domain;
2111         data.iommu = iommu;
2112
2113         return pci_for_each_dma_alias(to_pci_dev(dev),
2114                                       &domain_context_mapping_cb, &data);
2115 }
2116
2117 static int domain_context_mapped_cb(struct pci_dev *pdev,
2118                                     u16 alias, void *opaque)
2119 {
2120         struct intel_iommu *iommu = opaque;
2121
2122         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2123 }
2124
2125 static int domain_context_mapped(struct device *dev)
2126 {
2127         struct intel_iommu *iommu;
2128         u8 bus, devfn;
2129
2130         iommu = device_to_iommu(dev, &bus, &devfn);
2131         if (!iommu)
2132                 return -ENODEV;
2133
2134         if (!dev_is_pci(dev))
2135                 return device_context_mapped(iommu, bus, devfn);
2136
2137         return !pci_for_each_dma_alias(to_pci_dev(dev),
2138                                        domain_context_mapped_cb, iommu);
2139 }
2140
2141 /* Returns a number of VTD pages, but aligned to MM page size */
2142 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2143                                             size_t size)
2144 {
2145         host_addr &= ~PAGE_MASK;
2146         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2147 }
2148
2149 /* Return largest possible superpage level for a given mapping */
2150 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2151                                           unsigned long iov_pfn,
2152                                           unsigned long phy_pfn,
2153                                           unsigned long pages)
2154 {
2155         int support, level = 1;
2156         unsigned long pfnmerge;
2157
2158         support = domain->iommu_superpage;
2159
2160         /* To use a large page, the virtual *and* physical addresses
2161            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2162            of them will mean we have to use smaller pages. So just
2163            merge them and check both at once. */
2164         pfnmerge = iov_pfn | phy_pfn;
2165
2166         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2167                 pages >>= VTD_STRIDE_SHIFT;
2168                 if (!pages)
2169                         break;
2170                 pfnmerge >>= VTD_STRIDE_SHIFT;
2171                 level++;
2172                 support--;
2173         }
2174         return level;
2175 }
2176
2177 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178                             struct scatterlist *sg, unsigned long phys_pfn,
2179                             unsigned long nr_pages, int prot)
2180 {
2181         struct dma_pte *first_pte = NULL, *pte = NULL;
2182         phys_addr_t uninitialized_var(pteval);
2183         unsigned long sg_res = 0;
2184         unsigned int largepage_lvl = 0;
2185         unsigned long lvl_pages = 0;
2186
2187         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2188
2189         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2190                 return -EINVAL;
2191
2192         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2193
2194         if (!sg) {
2195                 sg_res = nr_pages;
2196                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2197         }
2198
2199         while (nr_pages > 0) {
2200                 uint64_t tmp;
2201
2202                 if (!sg_res) {
2203                         sg_res = aligned_nrpages(sg->offset, sg->length);
2204                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2205                         sg->dma_length = sg->length;
2206                         pteval = page_to_phys(sg_page(sg)) | prot;
2207                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2208                 }
2209
2210                 if (!pte) {
2211                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2212
2213                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2214                         if (!pte)
2215                                 return -ENOMEM;
2216                         /* It is large page*/
2217                         if (largepage_lvl > 1) {
2218                                 unsigned long nr_superpages, end_pfn;
2219
2220                                 pteval |= DMA_PTE_LARGE_PAGE;
2221                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2222
2223                                 nr_superpages = sg_res / lvl_pages;
2224                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2225
2226                                 /*
2227                                  * Ensure that old small page tables are
2228                                  * removed to make room for superpage(s).
2229                                  */
2230                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2231                         } else {
2232                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2233                         }
2234
2235                 }
2236                 /* We don't need lock here, nobody else
2237                  * touches the iova range
2238                  */
2239                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2240                 if (tmp) {
2241                         static int dumps = 5;
2242                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2243                                 iov_pfn, tmp, (unsigned long long)pteval);
2244                         if (dumps) {
2245                                 dumps--;
2246                                 debug_dma_dump_mappings(NULL);
2247                         }
2248                         WARN_ON(1);
2249                 }
2250
2251                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2252
2253                 BUG_ON(nr_pages < lvl_pages);
2254                 BUG_ON(sg_res < lvl_pages);
2255
2256                 nr_pages -= lvl_pages;
2257                 iov_pfn += lvl_pages;
2258                 phys_pfn += lvl_pages;
2259                 pteval += lvl_pages * VTD_PAGE_SIZE;
2260                 sg_res -= lvl_pages;
2261
2262                 /* If the next PTE would be the first in a new page, then we
2263                    need to flush the cache on the entries we've just written.
2264                    And then we'll need to recalculate 'pte', so clear it and
2265                    let it get set again in the if (!pte) block above.
2266
2267                    If we're done (!nr_pages) we need to flush the cache too.
2268
2269                    Also if we've been setting superpages, we may need to
2270                    recalculate 'pte' and switch back to smaller pages for the
2271                    end of the mapping, if the trailing size is not enough to
2272                    use another superpage (i.e. sg_res < lvl_pages). */
2273                 pte++;
2274                 if (!nr_pages || first_pte_in_page(pte) ||
2275                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2276                         domain_flush_cache(domain, first_pte,
2277                                            (void *)pte - (void *)first_pte);
2278                         pte = NULL;
2279                 }
2280
2281                 if (!sg_res && nr_pages)
2282                         sg = sg_next(sg);
2283         }
2284         return 0;
2285 }
2286
2287 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2288                                     struct scatterlist *sg, unsigned long nr_pages,
2289                                     int prot)
2290 {
2291         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2292 }
2293
2294 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2295                                      unsigned long phys_pfn, unsigned long nr_pages,
2296                                      int prot)
2297 {
2298         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2299 }
2300
2301 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2302 {
2303         if (!iommu)
2304                 return;
2305
2306         clear_context_table(iommu, bus, devfn);
2307         iommu->flush.flush_context(iommu, 0, 0, 0,
2308                                            DMA_CCMD_GLOBAL_INVL);
2309         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2310 }
2311
2312 static inline void unlink_domain_info(struct device_domain_info *info)
2313 {
2314         assert_spin_locked(&device_domain_lock);
2315         list_del(&info->link);
2316         list_del(&info->global);
2317         if (info->dev)
2318                 info->dev->archdata.iommu = NULL;
2319 }
2320
2321 static void domain_remove_dev_info(struct dmar_domain *domain)
2322 {
2323         struct device_domain_info *info, *tmp;
2324         unsigned long flags;
2325
2326         spin_lock_irqsave(&device_domain_lock, flags);
2327         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2328                 __dmar_remove_one_dev_info(info);
2329         spin_unlock_irqrestore(&device_domain_lock, flags);
2330 }
2331
2332 /*
2333  * find_domain
2334  * Note: we use struct device->archdata.iommu stores the info
2335  */
2336 static struct dmar_domain *find_domain(struct device *dev)
2337 {
2338         struct device_domain_info *info;
2339
2340         /* No lock here, assumes no domain exit in normal case */
2341         info = dev->archdata.iommu;
2342         if (info)
2343                 return info->domain;
2344         return NULL;
2345 }
2346
2347 static inline struct device_domain_info *
2348 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2349 {
2350         struct device_domain_info *info;
2351
2352         list_for_each_entry(info, &device_domain_list, global)
2353                 if (info->iommu->segment == segment && info->bus == bus &&
2354                     info->devfn == devfn)
2355                         return info;
2356
2357         return NULL;
2358 }
2359
2360 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2361                                                     int bus, int devfn,
2362                                                     struct device *dev,
2363                                                     struct dmar_domain *domain)
2364 {
2365         struct dmar_domain *found = NULL;
2366         struct device_domain_info *info;
2367         unsigned long flags;
2368         int ret;
2369
2370         info = alloc_devinfo_mem();
2371         if (!info)
2372                 return NULL;
2373
2374         info->bus = bus;
2375         info->devfn = devfn;
2376         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2377         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2378         info->ats_qdep = 0;
2379         info->dev = dev;
2380         info->domain = domain;
2381         info->iommu = iommu;
2382
2383         if (dev && dev_is_pci(dev)) {
2384                 struct pci_dev *pdev = to_pci_dev(info->dev);
2385
2386                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2387                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2388                     dmar_find_matched_atsr_unit(pdev))
2389                         info->ats_supported = 1;
2390
2391                 if (ecs_enabled(iommu)) {
2392                         if (pasid_enabled(iommu)) {
2393                                 int features = pci_pasid_features(pdev);
2394                                 if (features >= 0)
2395                                         info->pasid_supported = features | 1;
2396                         }
2397
2398                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2399                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2400                                 info->pri_supported = 1;
2401                 }
2402         }
2403
2404         spin_lock_irqsave(&device_domain_lock, flags);
2405         if (dev)
2406                 found = find_domain(dev);
2407
2408         if (!found) {
2409                 struct device_domain_info *info2;
2410                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2411                 if (info2) {
2412                         found      = info2->domain;
2413                         info2->dev = dev;
2414                 }
2415         }
2416
2417         if (found) {
2418                 spin_unlock_irqrestore(&device_domain_lock, flags);
2419                 free_devinfo_mem(info);
2420                 /* Caller must free the original domain */
2421                 return found;
2422         }
2423
2424         spin_lock(&iommu->lock);
2425         ret = domain_attach_iommu(domain, iommu);
2426         spin_unlock(&iommu->lock);
2427
2428         if (ret) {
2429                 spin_unlock_irqrestore(&device_domain_lock, flags);
2430                 free_devinfo_mem(info);
2431                 return NULL;
2432         }
2433
2434         list_add(&info->link, &domain->devices);
2435         list_add(&info->global, &device_domain_list);
2436         if (dev)
2437                 dev->archdata.iommu = info;
2438         spin_unlock_irqrestore(&device_domain_lock, flags);
2439
2440         if (dev && domain_context_mapping(domain, dev)) {
2441                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2442                 dmar_remove_one_dev_info(domain, dev);
2443                 return NULL;
2444         }
2445
2446         return domain;
2447 }
2448
2449 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2450 {
2451         *(u16 *)opaque = alias;
2452         return 0;
2453 }
2454
2455 /* domain is initialized */
2456 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2457 {
2458         struct device_domain_info *info = NULL;
2459         struct dmar_domain *domain, *tmp;
2460         struct intel_iommu *iommu;
2461         u16 req_id, dma_alias;
2462         unsigned long flags;
2463         u8 bus, devfn;
2464
2465         domain = find_domain(dev);
2466         if (domain)
2467                 return domain;
2468
2469         iommu = device_to_iommu(dev, &bus, &devfn);
2470         if (!iommu)
2471                 return NULL;
2472
2473         req_id = ((u16)bus << 8) | devfn;
2474
2475         if (dev_is_pci(dev)) {
2476                 struct pci_dev *pdev = to_pci_dev(dev);
2477
2478                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2479
2480                 spin_lock_irqsave(&device_domain_lock, flags);
2481                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2482                                                       PCI_BUS_NUM(dma_alias),
2483                                                       dma_alias & 0xff);
2484                 if (info) {
2485                         iommu = info->iommu;
2486                         domain = info->domain;
2487                 }
2488                 spin_unlock_irqrestore(&device_domain_lock, flags);
2489
2490                 /* DMA alias already has a domain, uses it */
2491                 if (info)
2492                         goto found_domain;
2493         }
2494
2495         /* Allocate and initialize new domain for the device */
2496         domain = alloc_domain(0);
2497         if (!domain)
2498                 return NULL;
2499         if (domain_init(domain, iommu, gaw)) {
2500                 domain_exit(domain);
2501                 return NULL;
2502         }
2503
2504         /* register PCI DMA alias device */
2505         if (dev_is_pci(dev) && req_id != dma_alias) {
2506                 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2507                                                dma_alias & 0xff, NULL, domain);
2508
2509                 if (!tmp || tmp != domain) {
2510                         domain_exit(domain);
2511                         domain = tmp;
2512                 }
2513
2514                 if (!domain)
2515                         return NULL;
2516         }
2517
2518 found_domain:
2519         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2520
2521         if (!tmp || tmp != domain) {
2522                 domain_exit(domain);
2523                 domain = tmp;
2524         }
2525
2526         return domain;
2527 }
2528
2529 static int iommu_domain_identity_map(struct dmar_domain *domain,
2530                                      unsigned long long start,
2531                                      unsigned long long end)
2532 {
2533         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2534         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2535
2536         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2537                           dma_to_mm_pfn(last_vpfn))) {
2538                 pr_err("Reserving iova failed\n");
2539                 return -ENOMEM;
2540         }
2541
2542         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2543         /*
2544          * RMRR range might have overlap with physical memory range,
2545          * clear it first
2546          */
2547         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2548
2549         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2550                                   last_vpfn - first_vpfn + 1,
2551                                   DMA_PTE_READ|DMA_PTE_WRITE);
2552 }
2553
2554 static int domain_prepare_identity_map(struct device *dev,
2555                                        struct dmar_domain *domain,
2556                                        unsigned long long start,
2557                                        unsigned long long end)
2558 {
2559         /* For _hardware_ passthrough, don't bother. But for software
2560            passthrough, we do it anyway -- it may indicate a memory
2561            range which is reserved in E820, so which didn't get set
2562            up to start with in si_domain */
2563         if (domain == si_domain && hw_pass_through) {
2564                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2565                         dev_name(dev), start, end);
2566                 return 0;
2567         }
2568
2569         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2570                 dev_name(dev), start, end);
2571
2572         if (end < start) {
2573                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2574                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2575                         dmi_get_system_info(DMI_BIOS_VENDOR),
2576                         dmi_get_system_info(DMI_BIOS_VERSION),
2577                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2578                 return -EIO;
2579         }
2580
2581         if (end >> agaw_to_width(domain->agaw)) {
2582                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2583                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2584                      agaw_to_width(domain->agaw),
2585                      dmi_get_system_info(DMI_BIOS_VENDOR),
2586                      dmi_get_system_info(DMI_BIOS_VERSION),
2587                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2588                 return -EIO;
2589         }
2590
2591         return iommu_domain_identity_map(domain, start, end);
2592 }
2593
2594 static int iommu_prepare_identity_map(struct device *dev,
2595                                       unsigned long long start,
2596                                       unsigned long long end)
2597 {
2598         struct dmar_domain *domain;
2599         int ret;
2600
2601         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2602         if (!domain)
2603                 return -ENOMEM;
2604
2605         ret = domain_prepare_identity_map(dev, domain, start, end);
2606         if (ret)
2607                 domain_exit(domain);
2608
2609         return ret;
2610 }
2611
2612 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2613                                          struct device *dev)
2614 {
2615         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2616                 return 0;
2617         return iommu_prepare_identity_map(dev, rmrr->base_address,
2618                                           rmrr->end_address);
2619 }
2620
2621 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2622 static inline void iommu_prepare_isa(void)
2623 {
2624         struct pci_dev *pdev;
2625         int ret;
2626
2627         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2628         if (!pdev)
2629                 return;
2630
2631         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2632         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2633
2634         if (ret)
2635                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2636
2637         pci_dev_put(pdev);
2638 }
2639 #else
2640 static inline void iommu_prepare_isa(void)
2641 {
2642         return;
2643 }
2644 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2645
2646 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2647
2648 static int __init si_domain_init(int hw)
2649 {
2650         int nid, ret = 0;
2651
2652         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2653         if (!si_domain)
2654                 return -EFAULT;
2655
2656         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2657                 domain_exit(si_domain);
2658                 return -EFAULT;
2659         }
2660
2661         pr_debug("Identity mapping domain allocated\n");
2662
2663         if (hw)
2664                 return 0;
2665
2666         for_each_online_node(nid) {
2667                 unsigned long start_pfn, end_pfn;
2668                 int i;
2669
2670                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2671                         ret = iommu_domain_identity_map(si_domain,
2672                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2673                         if (ret)
2674                                 return ret;
2675                 }
2676         }
2677
2678         return 0;
2679 }
2680
2681 static int identity_mapping(struct device *dev)
2682 {
2683         struct device_domain_info *info;
2684
2685         if (likely(!iommu_identity_mapping))
2686                 return 0;
2687
2688         info = dev->archdata.iommu;
2689         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2690                 return (info->domain == si_domain);
2691
2692         return 0;
2693 }
2694
2695 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2696 {
2697         struct dmar_domain *ndomain;
2698         struct intel_iommu *iommu;
2699         u8 bus, devfn;
2700
2701         iommu = device_to_iommu(dev, &bus, &devfn);
2702         if (!iommu)
2703                 return -ENODEV;
2704
2705         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2706         if (ndomain != domain)
2707                 return -EBUSY;
2708
2709         return 0;
2710 }
2711
2712 static bool device_has_rmrr(struct device *dev)
2713 {
2714         struct dmar_rmrr_unit *rmrr;
2715         struct device *tmp;
2716         int i;
2717
2718         rcu_read_lock();
2719         for_each_rmrr_units(rmrr) {
2720                 /*
2721                  * Return TRUE if this RMRR contains the device that
2722                  * is passed in.
2723                  */
2724                 for_each_active_dev_scope(rmrr->devices,
2725                                           rmrr->devices_cnt, i, tmp)
2726                         if (tmp == dev) {
2727                                 rcu_read_unlock();
2728                                 return true;
2729                         }
2730         }
2731         rcu_read_unlock();
2732         return false;
2733 }
2734
2735 /*
2736  * There are a couple cases where we need to restrict the functionality of
2737  * devices associated with RMRRs.  The first is when evaluating a device for
2738  * identity mapping because problems exist when devices are moved in and out
2739  * of domains and their respective RMRR information is lost.  This means that
2740  * a device with associated RMRRs will never be in a "passthrough" domain.
2741  * The second is use of the device through the IOMMU API.  This interface
2742  * expects to have full control of the IOVA space for the device.  We cannot
2743  * satisfy both the requirement that RMRR access is maintained and have an
2744  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2745  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2746  * We therefore prevent devices associated with an RMRR from participating in
2747  * the IOMMU API, which eliminates them from device assignment.
2748  *
2749  * In both cases we assume that PCI USB devices with RMRRs have them largely
2750  * for historical reasons and that the RMRR space is not actively used post
2751  * boot.  This exclusion may change if vendors begin to abuse it.
2752  *
2753  * The same exception is made for graphics devices, with the requirement that
2754  * any use of the RMRR regions will be torn down before assigning the device
2755  * to a guest.
2756  */
2757 static bool device_is_rmrr_locked(struct device *dev)
2758 {
2759         if (!device_has_rmrr(dev))
2760                 return false;
2761
2762         if (dev_is_pci(dev)) {
2763                 struct pci_dev *pdev = to_pci_dev(dev);
2764
2765                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2766                         return false;
2767         }
2768
2769         return true;
2770 }
2771
2772 static int iommu_should_identity_map(struct device *dev, int startup)
2773 {
2774
2775         if (dev_is_pci(dev)) {
2776                 struct pci_dev *pdev = to_pci_dev(dev);
2777
2778                 if (device_is_rmrr_locked(dev))
2779                         return 0;
2780
2781                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2782                         return 1;
2783
2784                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2785                         return 1;
2786
2787                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2788                         return 0;
2789
2790                 /*
2791                  * We want to start off with all devices in the 1:1 domain, and
2792                  * take them out later if we find they can't access all of memory.
2793                  *
2794                  * However, we can't do this for PCI devices behind bridges,
2795                  * because all PCI devices behind the same bridge will end up
2796                  * with the same source-id on their transactions.
2797                  *
2798                  * Practically speaking, we can't change things around for these
2799                  * devices at run-time, because we can't be sure there'll be no
2800                  * DMA transactions in flight for any of their siblings.
2801                  *
2802                  * So PCI devices (unless they're on the root bus) as well as
2803                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2804                  * the 1:1 domain, just in _case_ one of their siblings turns out
2805                  * not to be able to map all of memory.
2806                  */
2807                 if (!pci_is_pcie(pdev)) {
2808                         if (!pci_is_root_bus(pdev->bus))
2809                                 return 0;
2810                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2811                                 return 0;
2812                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2813                         return 0;
2814         } else {
2815                 if (device_has_rmrr(dev))
2816                         return 0;
2817         }
2818
2819         /*
2820          * At boot time, we don't yet know if devices will be 64-bit capable.
2821          * Assume that they will — if they turn out not to be, then we can
2822          * take them out of the 1:1 domain later.
2823          */
2824         if (!startup) {
2825                 /*
2826                  * If the device's dma_mask is less than the system's memory
2827                  * size then this is not a candidate for identity mapping.
2828                  */
2829                 u64 dma_mask = *dev->dma_mask;
2830
2831                 if (dev->coherent_dma_mask &&
2832                     dev->coherent_dma_mask < dma_mask)
2833                         dma_mask = dev->coherent_dma_mask;
2834
2835                 return dma_mask >= dma_get_required_mask(dev);
2836         }
2837
2838         return 1;
2839 }
2840
2841 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2842 {
2843         int ret;
2844
2845         if (!iommu_should_identity_map(dev, 1))
2846                 return 0;
2847
2848         ret = domain_add_dev_info(si_domain, dev);
2849         if (!ret)
2850                 pr_info("%s identity mapping for device %s\n",
2851                         hw ? "Hardware" : "Software", dev_name(dev));
2852         else if (ret == -ENODEV)
2853                 /* device not associated with an iommu */
2854                 ret = 0;
2855
2856         return ret;
2857 }
2858
2859
2860 static int __init iommu_prepare_static_identity_mapping(int hw)
2861 {
2862         struct pci_dev *pdev = NULL;
2863         struct dmar_drhd_unit *drhd;
2864         struct intel_iommu *iommu;
2865         struct device *dev;
2866         int i;
2867         int ret = 0;
2868
2869         for_each_pci_dev(pdev) {
2870                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2871                 if (ret)
2872                         return ret;
2873         }
2874
2875         for_each_active_iommu(iommu, drhd)
2876                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2877                         struct acpi_device_physical_node *pn;
2878                         struct acpi_device *adev;
2879
2880                         if (dev->bus != &acpi_bus_type)
2881                                 continue;
2882
2883                         adev= to_acpi_device(dev);
2884                         mutex_lock(&adev->physical_node_lock);
2885                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2886                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2887                                 if (ret)
2888                                         break;
2889                         }
2890                         mutex_unlock(&adev->physical_node_lock);
2891                         if (ret)
2892                                 return ret;
2893                 }
2894
2895         return 0;
2896 }
2897
2898 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2899 {
2900         /*
2901          * Start from the sane iommu hardware state.
2902          * If the queued invalidation is already initialized by us
2903          * (for example, while enabling interrupt-remapping) then
2904          * we got the things already rolling from a sane state.
2905          */
2906         if (!iommu->qi) {
2907                 /*
2908                  * Clear any previous faults.
2909                  */
2910                 dmar_fault(-1, iommu);
2911                 /*
2912                  * Disable queued invalidation if supported and already enabled
2913                  * before OS handover.
2914                  */
2915                 dmar_disable_qi(iommu);
2916         }
2917
2918         if (dmar_enable_qi(iommu)) {
2919                 /*
2920                  * Queued Invalidate not enabled, use Register Based Invalidate
2921                  */
2922                 iommu->flush.flush_context = __iommu_flush_context;
2923                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2924                 pr_info("%s: Using Register based invalidation\n",
2925                         iommu->name);
2926         } else {
2927                 iommu->flush.flush_context = qi_flush_context;
2928                 iommu->flush.flush_iotlb = qi_flush_iotlb;
2929                 pr_info("%s: Using Queued invalidation\n", iommu->name);
2930         }
2931 }
2932
2933 static int copy_context_table(struct intel_iommu *iommu,
2934                               struct root_entry *old_re,
2935                               struct context_entry **tbl,
2936                               int bus, bool ext)
2937 {
2938         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2939         struct context_entry *new_ce = NULL, ce;
2940         struct context_entry *old_ce = NULL;
2941         struct root_entry re;
2942         phys_addr_t old_ce_phys;
2943
2944         tbl_idx = ext ? bus * 2 : bus;
2945         memcpy(&re, old_re, sizeof(re));
2946
2947         for (devfn = 0; devfn < 256; devfn++) {
2948                 /* First calculate the correct index */
2949                 idx = (ext ? devfn * 2 : devfn) % 256;
2950
2951                 if (idx == 0) {
2952                         /* First save what we may have and clean up */
2953                         if (new_ce) {
2954                                 tbl[tbl_idx] = new_ce;
2955                                 __iommu_flush_cache(iommu, new_ce,
2956                                                     VTD_PAGE_SIZE);
2957                                 pos = 1;
2958                         }
2959
2960                         if (old_ce)
2961                                 iounmap(old_ce);
2962
2963                         ret = 0;
2964                         if (devfn < 0x80)
2965                                 old_ce_phys = root_entry_lctp(&re);
2966                         else
2967                                 old_ce_phys = root_entry_uctp(&re);
2968
2969                         if (!old_ce_phys) {
2970                                 if (ext && devfn == 0) {
2971                                         /* No LCTP, try UCTP */
2972                                         devfn = 0x7f;
2973                                         continue;
2974                                 } else {
2975                                         goto out;
2976                                 }
2977                         }
2978
2979                         ret = -ENOMEM;
2980                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
2981                                         MEMREMAP_WB);
2982                         if (!old_ce)
2983                                 goto out;
2984
2985                         new_ce = alloc_pgtable_page(iommu->node);
2986                         if (!new_ce)
2987                                 goto out_unmap;
2988
2989                         ret = 0;
2990                 }
2991
2992                 /* Now copy the context entry */
2993                 memcpy(&ce, old_ce + idx, sizeof(ce));
2994
2995                 if (!__context_present(&ce))
2996                         continue;
2997
2998                 did = context_domain_id(&ce);
2999                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3000                         set_bit(did, iommu->domain_ids);
3001
3002                 /*
3003                  * We need a marker for copied context entries. This
3004                  * marker needs to work for the old format as well as
3005                  * for extended context entries.
3006                  *
3007                  * Bit 67 of the context entry is used. In the old
3008                  * format this bit is available to software, in the
3009                  * extended format it is the PGE bit, but PGE is ignored
3010                  * by HW if PASIDs are disabled (and thus still
3011                  * available).
3012                  *
3013                  * So disable PASIDs first and then mark the entry
3014                  * copied. This means that we don't copy PASID
3015                  * translations from the old kernel, but this is fine as
3016                  * faults there are not fatal.
3017                  */
3018                 context_clear_pasid_enable(&ce);
3019                 context_set_copied(&ce);
3020
3021                 new_ce[idx] = ce;
3022         }
3023
3024         tbl[tbl_idx + pos] = new_ce;
3025
3026         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3027
3028 out_unmap:
3029         memunmap(old_ce);
3030
3031 out:
3032         return ret;
3033 }
3034
3035 static int copy_translation_tables(struct intel_iommu *iommu)
3036 {
3037         struct context_entry **ctxt_tbls;
3038         struct root_entry *old_rt;
3039         phys_addr_t old_rt_phys;
3040         int ctxt_table_entries;
3041         unsigned long flags;
3042         u64 rtaddr_reg;
3043         int bus, ret;
3044         bool new_ext, ext;
3045
3046         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3047         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3048         new_ext    = !!ecap_ecs(iommu->ecap);
3049
3050         /*
3051          * The RTT bit can only be changed when translation is disabled,
3052          * but disabling translation means to open a window for data
3053          * corruption. So bail out and don't copy anything if we would
3054          * have to change the bit.
3055          */
3056         if (new_ext != ext)
3057                 return -EINVAL;
3058
3059         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3060         if (!old_rt_phys)
3061                 return -EINVAL;
3062
3063         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3064         if (!old_rt)
3065                 return -ENOMEM;
3066
3067         /* This is too big for the stack - allocate it from slab */
3068         ctxt_table_entries = ext ? 512 : 256;
3069         ret = -ENOMEM;
3070         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3071         if (!ctxt_tbls)
3072                 goto out_unmap;
3073
3074         for (bus = 0; bus < 256; bus++) {
3075                 ret = copy_context_table(iommu, &old_rt[bus],
3076                                          ctxt_tbls, bus, ext);
3077                 if (ret) {
3078                         pr_err("%s: Failed to copy context table for bus %d\n",
3079                                 iommu->name, bus);
3080                         continue;
3081                 }
3082         }
3083
3084         spin_lock_irqsave(&iommu->lock, flags);
3085
3086         /* Context tables are copied, now write them to the root_entry table */
3087         for (bus = 0; bus < 256; bus++) {
3088                 int idx = ext ? bus * 2 : bus;
3089                 u64 val;
3090
3091                 if (ctxt_tbls[idx]) {
3092                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3093                         iommu->root_entry[bus].lo = val;
3094                 }
3095
3096                 if (!ext || !ctxt_tbls[idx + 1])
3097                         continue;
3098
3099                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3100                 iommu->root_entry[bus].hi = val;
3101         }
3102
3103         spin_unlock_irqrestore(&iommu->lock, flags);
3104
3105         kfree(ctxt_tbls);
3106
3107         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3108
3109         ret = 0;
3110
3111 out_unmap:
3112         memunmap(old_rt);
3113
3114         return ret;
3115 }
3116
3117 static int __init init_dmars(void)
3118 {
3119         struct dmar_drhd_unit *drhd;
3120         struct dmar_rmrr_unit *rmrr;
3121         bool copied_tables = false;
3122         struct device *dev;
3123         struct intel_iommu *iommu;
3124         int i, ret, cpu;
3125
3126         /*
3127          * for each drhd
3128          *    allocate root
3129          *    initialize and program root entry to not present
3130          * endfor
3131          */
3132         for_each_drhd_unit(drhd) {
3133                 /*
3134                  * lock not needed as this is only incremented in the single
3135                  * threaded kernel __init code path all other access are read
3136                  * only
3137                  */
3138                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3139                         g_num_of_iommus++;
3140                         continue;
3141                 }
3142                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3143         }
3144
3145         /* Preallocate enough resources for IOMMU hot-addition */
3146         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3147                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3148
3149         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3150                         GFP_KERNEL);
3151         if (!g_iommus) {
3152                 pr_err("Allocating global iommu array failed\n");
3153                 ret = -ENOMEM;
3154                 goto error;
3155         }
3156
3157         for_each_possible_cpu(cpu) {
3158                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3159                                                               cpu);
3160
3161                 dfd->tables = kzalloc(g_num_of_iommus *
3162                                       sizeof(struct deferred_flush_table),
3163                                       GFP_KERNEL);
3164                 if (!dfd->tables) {
3165                         ret = -ENOMEM;
3166                         goto free_g_iommus;
3167                 }
3168
3169                 spin_lock_init(&dfd->lock);
3170                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3171         }
3172
3173         for_each_active_iommu(iommu, drhd) {
3174                 g_iommus[iommu->seq_id] = iommu;
3175
3176                 intel_iommu_init_qi(iommu);
3177
3178                 ret = iommu_init_domains(iommu);
3179                 if (ret)
3180                         goto free_iommu;
3181
3182                 init_translation_status(iommu);
3183
3184                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3185                         iommu_disable_translation(iommu);
3186                         clear_translation_pre_enabled(iommu);
3187                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3188                                 iommu->name);
3189                 }
3190
3191                 /*
3192                  * TBD:
3193                  * we could share the same root & context tables
3194                  * among all IOMMU's. Need to Split it later.
3195                  */
3196                 ret = iommu_alloc_root_entry(iommu);
3197                 if (ret)
3198                         goto free_iommu;
3199
3200                 if (translation_pre_enabled(iommu)) {
3201                         pr_info("Translation already enabled - trying to copy translation structures\n");
3202
3203                         ret = copy_translation_tables(iommu);
3204                         if (ret) {
3205                                 /*
3206                                  * We found the IOMMU with translation
3207                                  * enabled - but failed to copy over the
3208                                  * old root-entry table. Try to proceed
3209                                  * by disabling translation now and
3210                                  * allocating a clean root-entry table.
3211                                  * This might cause DMAR faults, but
3212                                  * probably the dump will still succeed.
3213                                  */
3214                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3215                                        iommu->name);
3216                                 iommu_disable_translation(iommu);
3217                                 clear_translation_pre_enabled(iommu);
3218                         } else {
3219                                 pr_info("Copied translation tables from previous kernel for %s\n",
3220                                         iommu->name);
3221                                 copied_tables = true;
3222                         }
3223                 }
3224
3225                 iommu_flush_write_buffer(iommu);
3226                 iommu_set_root_entry(iommu);
3227                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3228                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3229
3230                 if (!ecap_pass_through(iommu->ecap))
3231                         hw_pass_through = 0;
3232 #ifdef CONFIG_INTEL_IOMMU_SVM
3233                 if (pasid_enabled(iommu))
3234                         intel_svm_alloc_pasid_tables(iommu);
3235 #endif
3236         }
3237
3238         if (iommu_pass_through)
3239                 iommu_identity_mapping |= IDENTMAP_ALL;
3240
3241 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3242         iommu_identity_mapping |= IDENTMAP_GFX;
3243 #endif
3244
3245         if (iommu_identity_mapping) {
3246                 ret = si_domain_init(hw_pass_through);
3247                 if (ret)
3248                         goto free_iommu;
3249         }
3250
3251         check_tylersburg_isoch();
3252
3253         /*
3254          * If we copied translations from a previous kernel in the kdump
3255          * case, we can not assign the devices to domains now, as that
3256          * would eliminate the old mappings. So skip this part and defer
3257          * the assignment to device driver initialization time.
3258          */
3259         if (copied_tables)
3260                 goto domains_done;
3261
3262         /*
3263          * If pass through is not set or not enabled, setup context entries for
3264          * identity mappings for rmrr, gfx, and isa and may fall back to static
3265          * identity mapping if iommu_identity_mapping is set.
3266          */
3267         if (iommu_identity_mapping) {
3268                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3269                 if (ret) {
3270                         pr_crit("Failed to setup IOMMU pass-through\n");
3271                         goto free_iommu;
3272                 }
3273         }
3274         /*
3275          * For each rmrr
3276          *   for each dev attached to rmrr
3277          *   do
3278          *     locate drhd for dev, alloc domain for dev
3279          *     allocate free domain
3280          *     allocate page table entries for rmrr
3281          *     if context not allocated for bus
3282          *           allocate and init context
3283          *           set present in root table for this bus
3284          *     init context with domain, translation etc
3285          *    endfor
3286          * endfor
3287          */
3288         pr_info("Setting RMRR:\n");
3289         for_each_rmrr_units(rmrr) {
3290                 /* some BIOS lists non-exist devices in DMAR table. */
3291                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3292                                           i, dev) {
3293                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3294                         if (ret)
3295                                 pr_err("Mapping reserved region failed\n");
3296                 }
3297         }
3298
3299         iommu_prepare_isa();
3300
3301 domains_done:
3302
3303         /*
3304          * for each drhd
3305          *   enable fault log
3306          *   global invalidate context cache
3307          *   global invalidate iotlb
3308          *   enable translation
3309          */
3310         for_each_iommu(iommu, drhd) {
3311                 if (drhd->ignored) {
3312                         /*
3313                          * we always have to disable PMRs or DMA may fail on
3314                          * this device
3315                          */
3316                         if (force_on)
3317                                 iommu_disable_protect_mem_regions(iommu);
3318                         continue;
3319                 }
3320
3321                 iommu_flush_write_buffer(iommu);
3322
3323 #ifdef CONFIG_INTEL_IOMMU_SVM
3324                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3325                         ret = intel_svm_enable_prq(iommu);
3326                         if (ret)
3327                                 goto free_iommu;
3328                 }
3329 #endif
3330                 ret = dmar_set_interrupt(iommu);
3331                 if (ret)
3332                         goto free_iommu;
3333
3334                 if (!translation_pre_enabled(iommu))
3335                         iommu_enable_translation(iommu);
3336
3337                 iommu_disable_protect_mem_regions(iommu);
3338         }
3339
3340         return 0;
3341
3342 free_iommu:
3343         for_each_active_iommu(iommu, drhd) {
3344                 disable_dmar_iommu(iommu);
3345                 free_dmar_iommu(iommu);
3346         }
3347 free_g_iommus:
3348         for_each_possible_cpu(cpu)
3349                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3350         kfree(g_iommus);
3351 error:
3352         return ret;
3353 }
3354
3355 /* This takes a number of _MM_ pages, not VTD pages */
3356 static unsigned long intel_alloc_iova(struct device *dev,
3357                                      struct dmar_domain *domain,
3358                                      unsigned long nrpages, uint64_t dma_mask)
3359 {
3360         unsigned long iova_pfn = 0;
3361
3362         /* Restrict dma_mask to the width that the iommu can handle */
3363         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3364         /* Ensure we reserve the whole size-aligned region */
3365         nrpages = __roundup_pow_of_two(nrpages);
3366
3367         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3368                 /*
3369                  * First try to allocate an io virtual address in
3370                  * DMA_BIT_MASK(32) and if that fails then try allocating
3371                  * from higher range
3372                  */
3373                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3374                                            IOVA_PFN(DMA_BIT_MASK(32)));
3375                 if (iova_pfn)
3376                         return iova_pfn;
3377         }
3378         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3379         if (unlikely(!iova_pfn)) {
3380                 pr_err("Allocating %ld-page iova for %s failed",
3381                        nrpages, dev_name(dev));
3382                 return 0;
3383         }
3384
3385         return iova_pfn;
3386 }
3387
3388 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3389 {
3390         struct dmar_rmrr_unit *rmrr;
3391         struct dmar_domain *domain;
3392         struct device *i_dev;
3393         int i, ret;
3394
3395         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3396         if (!domain) {
3397                 pr_err("Allocating domain for %s failed\n",
3398                        dev_name(dev));
3399                 return NULL;
3400         }
3401
3402         /* We have a new domain - setup possible RMRRs for the device */
3403         rcu_read_lock();
3404         for_each_rmrr_units(rmrr) {
3405                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3406                                           i, i_dev) {
3407                         if (i_dev != dev)
3408                                 continue;
3409
3410                         ret = domain_prepare_identity_map(dev, domain,
3411                                                           rmrr->base_address,
3412                                                           rmrr->end_address);
3413                         if (ret)
3414                                 dev_err(dev, "Mapping reserved region failed\n");
3415                 }
3416         }
3417         rcu_read_unlock();
3418
3419         return domain;
3420 }
3421
3422 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3423 {
3424         struct device_domain_info *info;
3425
3426         /* No lock here, assumes no domain exit in normal case */
3427         info = dev->archdata.iommu;
3428         if (likely(info))
3429                 return info->domain;
3430
3431         return __get_valid_domain_for_dev(dev);
3432 }
3433
3434 /* Check if the dev needs to go through non-identity map and unmap process.*/
3435 static int iommu_no_mapping(struct device *dev)
3436 {
3437         int found;
3438
3439         if (iommu_dummy(dev))
3440                 return 1;
3441
3442         if (!iommu_identity_mapping)
3443                 return 0;
3444
3445         found = identity_mapping(dev);
3446         if (found) {
3447                 if (iommu_should_identity_map(dev, 0))
3448                         return 1;
3449                 else {
3450                         /*
3451                          * 32 bit DMA is removed from si_domain and fall back
3452                          * to non-identity mapping.
3453                          */
3454                         dmar_remove_one_dev_info(si_domain, dev);
3455                         pr_info("32bit %s uses non-identity mapping\n",
3456                                 dev_name(dev));
3457                         return 0;
3458                 }
3459         } else {
3460                 /*
3461                  * In case of a detached 64 bit DMA device from vm, the device
3462                  * is put into si_domain for identity mapping.
3463                  */
3464                 if (iommu_should_identity_map(dev, 0)) {
3465                         int ret;
3466                         ret = domain_add_dev_info(si_domain, dev);
3467                         if (!ret) {
3468                                 pr_info("64bit %s uses identity mapping\n",
3469                                         dev_name(dev));
3470                                 return 1;
3471                         }
3472                 }
3473         }
3474
3475         return 0;
3476 }
3477
3478 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3479                                      size_t size, int dir, u64 dma_mask)
3480 {
3481         struct dmar_domain *domain;
3482         phys_addr_t start_paddr;
3483         unsigned long iova_pfn;
3484         int prot = 0;
3485         int ret;
3486         struct intel_iommu *iommu;
3487         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3488
3489         BUG_ON(dir == DMA_NONE);
3490
3491         if (iommu_no_mapping(dev))
3492                 return paddr;
3493
3494         domain = get_valid_domain_for_dev(dev);
3495         if (!domain)
3496                 return 0;
3497
3498         iommu = domain_get_iommu(domain);
3499         size = aligned_nrpages(paddr, size);
3500
3501         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3502         if (!iova_pfn)
3503                 goto error;
3504
3505         /*
3506          * Check if DMAR supports zero-length reads on write only
3507          * mappings..
3508          */
3509         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3510                         !cap_zlr(iommu->cap))
3511                 prot |= DMA_PTE_READ;
3512         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3513                 prot |= DMA_PTE_WRITE;
3514         /*
3515          * paddr - (paddr + size) might be partial page, we should map the whole
3516          * page.  Note: if two part of one page are separately mapped, we
3517          * might have two guest_addr mapping to the same host paddr, but this
3518          * is not a big problem
3519          */
3520         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3521                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3522         if (ret)
3523                 goto error;
3524
3525         /* it's a non-present to present mapping. Only flush if caching mode */
3526         if (cap_caching_mode(iommu->cap))
3527                 iommu_flush_iotlb_psi(iommu, domain,
3528                                       mm_to_dma_pfn(iova_pfn),
3529                                       size, 0, 1);
3530         else
3531                 iommu_flush_write_buffer(iommu);
3532
3533         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3534         start_paddr += paddr & ~PAGE_MASK;
3535         return start_paddr;
3536
3537 error:
3538         if (iova_pfn)
3539                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3540         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3541                 dev_name(dev), size, (unsigned long long)paddr, dir);
3542         return 0;
3543 }
3544
3545 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3546                                  unsigned long offset, size_t size,
3547                                  enum dma_data_direction dir,
3548                                  struct dma_attrs *attrs)
3549 {
3550         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3551                                   dir, *dev->dma_mask);
3552 }
3553
3554 static void flush_unmaps(struct deferred_flush_data *flush_data)
3555 {
3556         int i, j;
3557
3558         flush_data->timer_on = 0;
3559
3560         /* just flush them all */
3561         for (i = 0; i < g_num_of_iommus; i++) {
3562                 struct intel_iommu *iommu = g_iommus[i];
3563                 struct deferred_flush_table *flush_table =
3564                                 &flush_data->tables[i];
3565                 if (!iommu)
3566                         continue;
3567
3568                 if (!flush_table->next)
3569                         continue;
3570
3571                 /* In caching mode, global flushes turn emulation expensive */
3572                 if (!cap_caching_mode(iommu->cap))
3573                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3574                                          DMA_TLB_GLOBAL_FLUSH);
3575                 for (j = 0; j < flush_table->next; j++) {
3576                         unsigned long mask;
3577                         struct deferred_flush_entry *entry =
3578                                                 &flush_table->entries[j];
3579                         unsigned long iova_pfn = entry->iova_pfn;
3580                         unsigned long nrpages = entry->nrpages;
3581                         struct dmar_domain *domain = entry->domain;
3582                         struct page *freelist = entry->freelist;
3583
3584                         /* On real hardware multiple invalidations are expensive */
3585                         if (cap_caching_mode(iommu->cap))
3586                                 iommu_flush_iotlb_psi(iommu, domain,
3587                                         mm_to_dma_pfn(iova_pfn),
3588                                         nrpages, !freelist, 0);
3589                         else {
3590                                 mask = ilog2(nrpages);
3591                                 iommu_flush_dev_iotlb(domain,
3592                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3593                         }
3594                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3595                         if (freelist)
3596                                 dma_free_pagelist(freelist);
3597                 }
3598                 flush_table->next = 0;
3599         }
3600
3601         flush_data->size = 0;
3602 }
3603
3604 static void flush_unmaps_timeout(unsigned long cpuid)
3605 {
3606         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3607         unsigned long flags;
3608
3609         spin_lock_irqsave(&flush_data->lock, flags);
3610         flush_unmaps(flush_data);
3611         spin_unlock_irqrestore(&flush_data->lock, flags);
3612 }
3613
3614 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3615                       unsigned long nrpages, struct page *freelist)
3616 {
3617         unsigned long flags;
3618         int entry_id, iommu_id;
3619         struct intel_iommu *iommu;
3620         struct deferred_flush_entry *entry;
3621         struct deferred_flush_data *flush_data;
3622         unsigned int cpuid;
3623
3624         cpuid = get_cpu();
3625         flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3626
3627         /* Flush all CPUs' entries to avoid deferring too much.  If
3628          * this becomes a bottleneck, can just flush us, and rely on
3629          * flush timer for the rest.
3630          */
3631         if (flush_data->size == HIGH_WATER_MARK) {
3632                 int cpu;
3633
3634                 for_each_online_cpu(cpu)
3635                         flush_unmaps_timeout(cpu);
3636         }
3637
3638         spin_lock_irqsave(&flush_data->lock, flags);
3639
3640         iommu = domain_get_iommu(dom);
3641         iommu_id = iommu->seq_id;
3642
3643         entry_id = flush_data->tables[iommu_id].next;
3644         ++(flush_data->tables[iommu_id].next);
3645
3646         entry = &flush_data->tables[iommu_id].entries[entry_id];
3647         entry->domain = dom;
3648         entry->iova_pfn = iova_pfn;
3649         entry->nrpages = nrpages;
3650         entry->freelist = freelist;
3651
3652         if (!flush_data->timer_on) {
3653                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3654                 flush_data->timer_on = 1;
3655         }
3656         flush_data->size++;
3657         spin_unlock_irqrestore(&flush_data->lock, flags);
3658
3659         put_cpu();
3660 }
3661
3662 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3663 {
3664         struct dmar_domain *domain;
3665         unsigned long start_pfn, last_pfn;
3666         unsigned long nrpages;
3667         unsigned long iova_pfn;
3668         struct intel_iommu *iommu;
3669         struct page *freelist;
3670
3671         if (iommu_no_mapping(dev))
3672                 return;
3673
3674         domain = find_domain(dev);
3675         BUG_ON(!domain);
3676
3677         iommu = domain_get_iommu(domain);
3678
3679         iova_pfn = IOVA_PFN(dev_addr);
3680
3681         nrpages = aligned_nrpages(dev_addr, size);
3682         start_pfn = mm_to_dma_pfn(iova_pfn);
3683         last_pfn = start_pfn + nrpages - 1;
3684
3685         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3686                  dev_name(dev), start_pfn, last_pfn);
3687
3688         freelist = domain_unmap(domain, start_pfn, last_pfn);
3689
3690         if (intel_iommu_strict) {
3691                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3692                                       nrpages, !freelist, 0);
3693                 /* free iova */
3694                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3695                 dma_free_pagelist(freelist);
3696         } else {
3697                 add_unmap(domain, iova_pfn, nrpages, freelist);
3698                 /*
3699                  * queue up the release of the unmap to save the 1/6th of the
3700                  * cpu used up by the iotlb flush operation...
3701                  */
3702         }
3703 }
3704
3705 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3706                              size_t size, enum dma_data_direction dir,
3707                              struct dma_attrs *attrs)
3708 {
3709         intel_unmap(dev, dev_addr, size);
3710 }
3711
3712 static void *intel_alloc_coherent(struct device *dev, size_t size,
3713                                   dma_addr_t *dma_handle, gfp_t flags,
3714                                   struct dma_attrs *attrs)
3715 {
3716         struct page *page = NULL;
3717         int order;
3718
3719         size = PAGE_ALIGN(size);
3720         order = get_order(size);
3721
3722         if (!iommu_no_mapping(dev))
3723                 flags &= ~(GFP_DMA | GFP_DMA32);
3724         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3725                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3726                         flags |= GFP_DMA;
3727                 else
3728                         flags |= GFP_DMA32;
3729         }
3730
3731         if (gfpflags_allow_blocking(flags)) {
3732                 unsigned int count = size >> PAGE_SHIFT;
3733
3734                 page = dma_alloc_from_contiguous(dev, count, order);
3735                 if (page && iommu_no_mapping(dev) &&
3736                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3737                         dma_release_from_contiguous(dev, page, count);
3738                         page = NULL;
3739                 }
3740         }
3741
3742         if (!page)
3743                 page = alloc_pages(flags, order);
3744         if (!page)
3745                 return NULL;
3746         memset(page_address(page), 0, size);
3747
3748         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3749                                          DMA_BIDIRECTIONAL,
3750                                          dev->coherent_dma_mask);
3751         if (*dma_handle)
3752                 return page_address(page);
3753         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3754                 __free_pages(page, order);
3755
3756         return NULL;
3757 }
3758
3759 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3760                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3761 {
3762         int order;
3763         struct page *page = virt_to_page(vaddr);
3764
3765         size = PAGE_ALIGN(size);
3766         order = get_order(size);
3767
3768         intel_unmap(dev, dma_handle, size);
3769         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3770                 __free_pages(page, order);
3771 }
3772
3773 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3774                            int nelems, enum dma_data_direction dir,
3775                            struct dma_attrs *attrs)
3776 {
3777         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3778         unsigned long nrpages = 0;
3779         struct scatterlist *sg;
3780         int i;
3781
3782         for_each_sg(sglist, sg, nelems, i) {
3783                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3784         }
3785
3786         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3787 }
3788
3789 static int intel_nontranslate_map_sg(struct device *hddev,
3790         struct scatterlist *sglist, int nelems, int dir)
3791 {
3792         int i;
3793         struct scatterlist *sg;
3794
3795         for_each_sg(sglist, sg, nelems, i) {
3796                 BUG_ON(!sg_page(sg));
3797                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3798                 sg->dma_length = sg->length;
3799         }
3800         return nelems;
3801 }
3802
3803 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3804                         enum dma_data_direction dir, struct dma_attrs *attrs)
3805 {
3806         int i;
3807         struct dmar_domain *domain;
3808         size_t size = 0;
3809         int prot = 0;
3810         unsigned long iova_pfn;
3811         int ret;
3812         struct scatterlist *sg;
3813         unsigned long start_vpfn;
3814         struct intel_iommu *iommu;
3815
3816         BUG_ON(dir == DMA_NONE);
3817         if (iommu_no_mapping(dev))
3818                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3819
3820         domain = get_valid_domain_for_dev(dev);
3821         if (!domain)
3822                 return 0;
3823
3824         iommu = domain_get_iommu(domain);
3825
3826         for_each_sg(sglist, sg, nelems, i)
3827                 size += aligned_nrpages(sg->offset, sg->length);
3828
3829         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3830                                 *dev->dma_mask);
3831         if (!iova_pfn) {
3832                 sglist->dma_length = 0;
3833                 return 0;
3834         }
3835
3836         /*
3837          * Check if DMAR supports zero-length reads on write only
3838          * mappings..
3839          */
3840         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3841                         !cap_zlr(iommu->cap))
3842                 prot |= DMA_PTE_READ;
3843         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3844                 prot |= DMA_PTE_WRITE;
3845
3846         start_vpfn = mm_to_dma_pfn(iova_pfn);
3847
3848         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3849         if (unlikely(ret)) {
3850                 dma_pte_free_pagetable(domain, start_vpfn,
3851                                        start_vpfn + size - 1);
3852                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3853                 return 0;
3854         }
3855
3856         /* it's a non-present to present mapping. Only flush if caching mode */
3857         if (cap_caching_mode(iommu->cap))
3858                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3859         else
3860                 iommu_flush_write_buffer(iommu);
3861
3862         return nelems;
3863 }
3864
3865 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3866 {
3867         return !dma_addr;
3868 }
3869
3870 struct dma_map_ops intel_dma_ops = {
3871         .alloc = intel_alloc_coherent,
3872         .free = intel_free_coherent,
3873         .map_sg = intel_map_sg,
3874         .unmap_sg = intel_unmap_sg,
3875         .map_page = intel_map_page,
3876         .unmap_page = intel_unmap_page,
3877         .mapping_error = intel_mapping_error,
3878 };
3879
3880 static inline int iommu_domain_cache_init(void)
3881 {
3882         int ret = 0;
3883
3884         iommu_domain_cache = kmem_cache_create("iommu_domain",
3885                                          sizeof(struct dmar_domain),
3886                                          0,
3887                                          SLAB_HWCACHE_ALIGN,
3888
3889                                          NULL);
3890         if (!iommu_domain_cache) {
3891                 pr_err("Couldn't create iommu_domain cache\n");
3892                 ret = -ENOMEM;
3893         }
3894
3895         return ret;
3896 }
3897
3898 static inline int iommu_devinfo_cache_init(void)
3899 {
3900         int ret = 0;
3901
3902         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3903                                          sizeof(struct device_domain_info),
3904                                          0,
3905                                          SLAB_HWCACHE_ALIGN,
3906                                          NULL);
3907         if (!iommu_devinfo_cache) {
3908                 pr_err("Couldn't create devinfo cache\n");
3909                 ret = -ENOMEM;
3910         }
3911
3912         return ret;
3913 }
3914
3915 static int __init iommu_init_mempool(void)
3916 {
3917         int ret;
3918         ret = iova_cache_get();
3919         if (ret)
3920                 return ret;
3921
3922         ret = iommu_domain_cache_init();
3923         if (ret)
3924                 goto domain_error;
3925
3926         ret = iommu_devinfo_cache_init();
3927         if (!ret)
3928                 return ret;
3929
3930         kmem_cache_destroy(iommu_domain_cache);
3931 domain_error:
3932         iova_cache_put();
3933
3934         return -ENOMEM;
3935 }
3936
3937 static void __init iommu_exit_mempool(void)
3938 {
3939         kmem_cache_destroy(iommu_devinfo_cache);
3940         kmem_cache_destroy(iommu_domain_cache);
3941         iova_cache_put();
3942 }
3943
3944 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3945 {
3946         struct dmar_drhd_unit *drhd;
3947         u32 vtbar;
3948         int rc;
3949
3950         /* We know that this device on this chipset has its own IOMMU.
3951          * If we find it under a different IOMMU, then the BIOS is lying
3952          * to us. Hope that the IOMMU for this device is actually
3953          * disabled, and it needs no translation...
3954          */
3955         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3956         if (rc) {
3957                 /* "can't" happen */
3958                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3959                 return;
3960         }
3961         vtbar &= 0xffff0000;
3962
3963         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3964         drhd = dmar_find_matched_drhd_unit(pdev);
3965         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3966                             TAINT_FIRMWARE_WORKAROUND,
3967                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3968                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3969 }
3970 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3971
3972 static void __init init_no_remapping_devices(void)
3973 {
3974         struct dmar_drhd_unit *drhd;
3975         struct device *dev;
3976         int i;
3977
3978         for_each_drhd_unit(drhd) {
3979                 if (!drhd->include_all) {
3980                         for_each_active_dev_scope(drhd->devices,
3981                                                   drhd->devices_cnt, i, dev)
3982                                 break;
3983                         /* ignore DMAR unit if no devices exist */
3984                         if (i == drhd->devices_cnt)
3985                                 drhd->ignored = 1;
3986                 }
3987         }
3988
3989         for_each_active_drhd_unit(drhd) {
3990                 if (drhd->include_all)
3991                         continue;
3992
3993                 for_each_active_dev_scope(drhd->devices,
3994                                           drhd->devices_cnt, i, dev)
3995                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3996                                 break;
3997                 if (i < drhd->devices_cnt)
3998                         continue;
3999
4000                 /* This IOMMU has *only* gfx devices. Either bypass it or
4001                    set the gfx_mapped flag, as appropriate */
4002                 if (dmar_map_gfx) {
4003                         intel_iommu_gfx_mapped = 1;
4004                 } else {
4005                         drhd->ignored = 1;
4006                         for_each_active_dev_scope(drhd->devices,
4007                                                   drhd->devices_cnt, i, dev)
4008                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4009                 }
4010         }
4011 }
4012
4013 #ifdef CONFIG_SUSPEND
4014 static int init_iommu_hw(void)
4015 {
4016         struct dmar_drhd_unit *drhd;
4017         struct intel_iommu *iommu = NULL;
4018
4019         for_each_active_iommu(iommu, drhd)
4020                 if (iommu->qi)
4021                         dmar_reenable_qi(iommu);
4022
4023         for_each_iommu(iommu, drhd) {
4024                 if (drhd->ignored) {
4025                         /*
4026                          * we always have to disable PMRs or DMA may fail on
4027                          * this device
4028                          */
4029                         if (force_on)
4030                                 iommu_disable_protect_mem_regions(iommu);
4031                         continue;
4032                 }
4033
4034                 iommu_flush_write_buffer(iommu);
4035
4036                 iommu_set_root_entry(iommu);
4037
4038                 iommu->flush.flush_context(iommu, 0, 0, 0,
4039                                            DMA_CCMD_GLOBAL_INVL);
4040                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4041                 iommu_enable_translation(iommu);
4042                 iommu_disable_protect_mem_regions(iommu);
4043         }
4044
4045         return 0;
4046 }
4047
4048 static void iommu_flush_all(void)
4049 {
4050         struct dmar_drhd_unit *drhd;
4051         struct intel_iommu *iommu;
4052
4053         for_each_active_iommu(iommu, drhd) {
4054                 iommu->flush.flush_context(iommu, 0, 0, 0,
4055                                            DMA_CCMD_GLOBAL_INVL);
4056                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4057                                          DMA_TLB_GLOBAL_FLUSH);
4058         }
4059 }
4060
4061 static int iommu_suspend(void)
4062 {
4063         struct dmar_drhd_unit *drhd;
4064         struct intel_iommu *iommu = NULL;
4065         unsigned long flag;
4066
4067         for_each_active_iommu(iommu, drhd) {
4068                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4069                                                  GFP_ATOMIC);
4070                 if (!iommu->iommu_state)
4071                         goto nomem;
4072         }
4073
4074         iommu_flush_all();
4075
4076         for_each_active_iommu(iommu, drhd) {
4077                 iommu_disable_translation(iommu);
4078
4079                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4080
4081                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4082                         readl(iommu->reg + DMAR_FECTL_REG);
4083                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4084                         readl(iommu->reg + DMAR_FEDATA_REG);
4085                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4086                         readl(iommu->reg + DMAR_FEADDR_REG);
4087                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4088                         readl(iommu->reg + DMAR_FEUADDR_REG);
4089
4090                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4091         }
4092         return 0;
4093
4094 nomem:
4095         for_each_active_iommu(iommu, drhd)
4096                 kfree(iommu->iommu_state);
4097
4098         return -ENOMEM;
4099 }
4100
4101 static void iommu_resume(void)
4102 {
4103         struct dmar_drhd_unit *drhd;
4104         struct intel_iommu *iommu = NULL;
4105         unsigned long flag;
4106
4107         if (init_iommu_hw()) {
4108                 if (force_on)
4109                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4110                 else
4111                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4112                 return;
4113         }
4114
4115         for_each_active_iommu(iommu, drhd) {
4116
4117                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4118
4119                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4120                         iommu->reg + DMAR_FECTL_REG);
4121                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4122                         iommu->reg + DMAR_FEDATA_REG);
4123                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4124                         iommu->reg + DMAR_FEADDR_REG);
4125                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4126                         iommu->reg + DMAR_FEUADDR_REG);
4127
4128                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4129         }
4130
4131         for_each_active_iommu(iommu, drhd)
4132                 kfree(iommu->iommu_state);
4133 }
4134
4135 static struct syscore_ops iommu_syscore_ops = {
4136         .resume         = iommu_resume,
4137         .suspend        = iommu_suspend,
4138 };
4139
4140 static void __init init_iommu_pm_ops(void)
4141 {
4142         register_syscore_ops(&iommu_syscore_ops);
4143 }
4144
4145 #else
4146 static inline void init_iommu_pm_ops(void) {}
4147 #endif  /* CONFIG_PM */
4148
4149
4150 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4151 {
4152         struct acpi_dmar_reserved_memory *rmrr;
4153         struct dmar_rmrr_unit *rmrru;
4154
4155         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4156         if (!rmrru)
4157                 return -ENOMEM;
4158
4159         rmrru->hdr = header;
4160         rmrr = (struct acpi_dmar_reserved_memory *)header;
4161         rmrru->base_address = rmrr->base_address;
4162         rmrru->end_address = rmrr->end_address;
4163         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4164                                 ((void *)rmrr) + rmrr->header.length,
4165                                 &rmrru->devices_cnt);
4166         if (rmrru->devices_cnt && rmrru->devices == NULL) {
4167                 kfree(rmrru);
4168                 return -ENOMEM;
4169         }
4170
4171         list_add(&rmrru->list, &dmar_rmrr_units);
4172
4173         return 0;
4174 }
4175
4176 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4177 {
4178         struct dmar_atsr_unit *atsru;
4179         struct acpi_dmar_atsr *tmp;
4180
4181         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4182                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4183                 if (atsr->segment != tmp->segment)
4184                         continue;
4185                 if (atsr->header.length != tmp->header.length)
4186                         continue;
4187                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4188                         return atsru;
4189         }
4190
4191         return NULL;
4192 }
4193
4194 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4195 {
4196         struct acpi_dmar_atsr *atsr;
4197         struct dmar_atsr_unit *atsru;
4198
4199         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4200                 return 0;
4201
4202         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4203         atsru = dmar_find_atsr(atsr);
4204         if (atsru)
4205                 return 0;
4206
4207         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4208         if (!atsru)
4209                 return -ENOMEM;
4210
4211         /*
4212          * If memory is allocated from slab by ACPI _DSM method, we need to
4213          * copy the memory content because the memory buffer will be freed
4214          * on return.
4215          */
4216         atsru->hdr = (void *)(atsru + 1);
4217         memcpy(atsru->hdr, hdr, hdr->length);
4218         atsru->include_all = atsr->flags & 0x1;
4219         if (!atsru->include_all) {
4220                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4221                                 (void *)atsr + atsr->header.length,
4222                                 &atsru->devices_cnt);
4223                 if (atsru->devices_cnt && atsru->devices == NULL) {
4224                         kfree(atsru);
4225                         return -ENOMEM;
4226                 }
4227         }
4228
4229         list_add_rcu(&atsru->list, &dmar_atsr_units);
4230
4231         return 0;
4232 }
4233
4234 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4235 {
4236         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4237         kfree(atsru);
4238 }
4239
4240 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4241 {
4242         struct acpi_dmar_atsr *atsr;
4243         struct dmar_atsr_unit *atsru;
4244
4245         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4246         atsru = dmar_find_atsr(atsr);
4247         if (atsru) {
4248                 list_del_rcu(&atsru->list);
4249                 synchronize_rcu();
4250                 intel_iommu_free_atsr(atsru);
4251         }
4252
4253         return 0;
4254 }
4255
4256 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4257 {
4258         int i;
4259         struct device *dev;
4260         struct acpi_dmar_atsr *atsr;
4261         struct dmar_atsr_unit *atsru;
4262
4263         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4264         atsru = dmar_find_atsr(atsr);
4265         if (!atsru)
4266                 return 0;
4267
4268         if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4269                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4270                                           i, dev)
4271                         return -EBUSY;
4272
4273         return 0;
4274 }
4275
4276 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4277 {
4278         int sp, ret = 0;
4279         struct intel_iommu *iommu = dmaru->iommu;
4280
4281         if (g_iommus[iommu->seq_id])
4282                 return 0;
4283
4284         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4285                 pr_warn("%s: Doesn't support hardware pass through.\n",
4286                         iommu->name);
4287                 return -ENXIO;
4288         }
4289         if (!ecap_sc_support(iommu->ecap) &&
4290             domain_update_iommu_snooping(iommu)) {
4291                 pr_warn("%s: Doesn't support snooping.\n",
4292                         iommu->name);
4293                 return -ENXIO;
4294         }
4295         sp = domain_update_iommu_superpage(iommu) - 1;
4296         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4297                 pr_warn("%s: Doesn't support large page.\n",
4298                         iommu->name);
4299                 return -ENXIO;
4300         }
4301
4302         /*
4303          * Disable translation if already enabled prior to OS handover.
4304          */
4305         if (iommu->gcmd & DMA_GCMD_TE)
4306                 iommu_disable_translation(iommu);
4307
4308         g_iommus[iommu->seq_id] = iommu;
4309         ret = iommu_init_domains(iommu);
4310         if (ret == 0)
4311                 ret = iommu_alloc_root_entry(iommu);
4312         if (ret)
4313                 goto out;
4314
4315 #ifdef CONFIG_INTEL_IOMMU_SVM
4316         if (pasid_enabled(iommu))
4317                 intel_svm_alloc_pasid_tables(iommu);
4318 #endif
4319
4320         if (dmaru->ignored) {
4321                 /*
4322                  * we always have to disable PMRs or DMA may fail on this device
4323                  */
4324                 if (force_on)
4325                         iommu_disable_protect_mem_regions(iommu);
4326                 return 0;
4327         }
4328
4329         intel_iommu_init_qi(iommu);
4330         iommu_flush_write_buffer(iommu);
4331
4332 #ifdef CONFIG_INTEL_IOMMU_SVM
4333         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4334                 ret = intel_svm_enable_prq(iommu);
4335                 if (ret)
4336                         goto disable_iommu;
4337         }
4338 #endif
4339         ret = dmar_set_interrupt(iommu);
4340         if (ret)
4341                 goto disable_iommu;
4342
4343         iommu_set_root_entry(iommu);
4344         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4345         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4346         iommu_enable_translation(iommu);
4347
4348         iommu_disable_protect_mem_regions(iommu);
4349         return 0;
4350
4351 disable_iommu:
4352         disable_dmar_iommu(iommu);
4353 out:
4354         free_dmar_iommu(iommu);
4355         return ret;
4356 }
4357
4358 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4359 {
4360         int ret = 0;
4361         struct intel_iommu *iommu = dmaru->iommu;
4362
4363         if (!intel_iommu_enabled)
4364                 return 0;
4365         if (iommu == NULL)
4366                 return -EINVAL;
4367
4368         if (insert) {
4369                 ret = intel_iommu_add(dmaru);
4370         } else {
4371                 disable_dmar_iommu(iommu);
4372                 free_dmar_iommu(iommu);
4373         }
4374
4375         return ret;
4376 }
4377
4378 static void intel_iommu_free_dmars(void)
4379 {
4380         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4381         struct dmar_atsr_unit *atsru, *atsr_n;
4382
4383         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4384                 list_del(&rmrru->list);
4385                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4386                 kfree(rmrru);
4387         }
4388
4389         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4390                 list_del(&atsru->list);
4391                 intel_iommu_free_atsr(atsru);
4392         }
4393 }
4394
4395 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4396 {
4397         int i, ret = 1;
4398         struct pci_bus *bus;
4399         struct pci_dev *bridge = NULL;
4400         struct device *tmp;
4401         struct acpi_dmar_atsr *atsr;
4402         struct dmar_atsr_unit *atsru;
4403
4404         dev = pci_physfn(dev);
4405         for (bus = dev->bus; bus; bus = bus->parent) {
4406                 bridge = bus->self;
4407                 /* If it's an integrated device, allow ATS */
4408                 if (!bridge)
4409                         return 1;
4410                 /* Connected via non-PCIe: no ATS */
4411                 if (!pci_is_pcie(bridge) ||
4412                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4413                         return 0;
4414                 /* If we found the root port, look it up in the ATSR */
4415                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4416                         break;
4417         }
4418
4419         rcu_read_lock();
4420         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4421                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4422                 if (atsr->segment != pci_domain_nr(dev->bus))
4423                         continue;
4424
4425                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4426                         if (tmp == &bridge->dev)
4427                                 goto out;
4428
4429                 if (atsru->include_all)
4430                         goto out;
4431         }
4432         ret = 0;
4433 out:
4434         rcu_read_unlock();
4435
4436         return ret;
4437 }
4438
4439 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4440 {
4441         int ret = 0;
4442         struct dmar_rmrr_unit *rmrru;
4443         struct dmar_atsr_unit *atsru;
4444         struct acpi_dmar_atsr *atsr;
4445         struct acpi_dmar_reserved_memory *rmrr;
4446
4447         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4448                 return 0;
4449
4450         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4451                 rmrr = container_of(rmrru->hdr,
4452                                     struct acpi_dmar_reserved_memory, header);
4453                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4454                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4455                                 ((void *)rmrr) + rmrr->header.length,
4456                                 rmrr->segment, rmrru->devices,
4457                                 rmrru->devices_cnt);
4458                         if(ret < 0)
4459                                 return ret;
4460                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4461                         dmar_remove_dev_scope(info, rmrr->segment,
4462                                 rmrru->devices, rmrru->devices_cnt);
4463                 }
4464         }
4465
4466         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4467                 if (atsru->include_all)
4468                         continue;
4469
4470                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4471                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4472                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4473                                         (void *)atsr + atsr->header.length,
4474                                         atsr->segment, atsru->devices,
4475                                         atsru->devices_cnt);
4476                         if (ret > 0)
4477                                 break;
4478                         else if(ret < 0)
4479                                 return ret;
4480                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4481                         if (dmar_remove_dev_scope(info, atsr->segment,
4482                                         atsru->devices, atsru->devices_cnt))
4483                                 break;
4484                 }
4485         }
4486
4487         return 0;
4488 }
4489
4490 /*
4491  * Here we only respond to action of unbound device from driver.
4492  *
4493  * Added device is not attached to its DMAR domain here yet. That will happen
4494  * when mapping the device to iova.
4495  */
4496 static int device_notifier(struct notifier_block *nb,
4497                                   unsigned long action, void *data)
4498 {
4499         struct device *dev = data;
4500         struct dmar_domain *domain;
4501
4502         if (iommu_dummy(dev))
4503                 return 0;
4504
4505         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4506                 return 0;
4507
4508         domain = find_domain(dev);
4509         if (!domain)
4510                 return 0;
4511
4512         dmar_remove_one_dev_info(domain, dev);
4513         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4514                 domain_exit(domain);
4515
4516         return 0;
4517 }
4518
4519 static struct notifier_block device_nb = {
4520         .notifier_call = device_notifier,
4521 };
4522
4523 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4524                                        unsigned long val, void *v)
4525 {
4526         struct memory_notify *mhp = v;
4527         unsigned long long start, end;
4528         unsigned long start_vpfn, last_vpfn;
4529
4530         switch (val) {
4531         case MEM_GOING_ONLINE:
4532                 start = mhp->start_pfn << PAGE_SHIFT;
4533                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4534                 if (iommu_domain_identity_map(si_domain, start, end)) {
4535                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4536                                 start, end);
4537                         return NOTIFY_BAD;
4538                 }
4539                 break;
4540
4541         case MEM_OFFLINE:
4542         case MEM_CANCEL_ONLINE:
4543                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4544                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4545                 while (start_vpfn <= last_vpfn) {
4546                         struct iova *iova;
4547                         struct dmar_drhd_unit *drhd;
4548                         struct intel_iommu *iommu;
4549                         struct page *freelist;
4550
4551                         iova = find_iova(&si_domain->iovad, start_vpfn);
4552                         if (iova == NULL) {
4553                                 pr_debug("Failed get IOVA for PFN %lx\n",
4554                                          start_vpfn);
4555                                 break;
4556                         }
4557
4558                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4559                                                      start_vpfn, last_vpfn);
4560                         if (iova == NULL) {
4561                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4562                                         start_vpfn, last_vpfn);
4563                                 return NOTIFY_BAD;
4564                         }
4565
4566                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4567                                                iova->pfn_hi);
4568
4569                         rcu_read_lock();
4570                         for_each_active_iommu(iommu, drhd)
4571                                 iommu_flush_iotlb_psi(iommu, si_domain,
4572                                         iova->pfn_lo, iova_size(iova),
4573                                         !freelist, 0);
4574                         rcu_read_unlock();
4575                         dma_free_pagelist(freelist);
4576
4577                         start_vpfn = iova->pfn_hi + 1;
4578                         free_iova_mem(iova);
4579                 }
4580                 break;
4581         }
4582
4583         return NOTIFY_OK;
4584 }
4585
4586 static struct notifier_block intel_iommu_memory_nb = {
4587         .notifier_call = intel_iommu_memory_notifier,
4588         .priority = 0
4589 };
4590
4591 static void free_all_cpu_cached_iovas(unsigned int cpu)
4592 {
4593         int i;
4594
4595         for (i = 0; i < g_num_of_iommus; i++) {
4596                 struct intel_iommu *iommu = g_iommus[i];
4597                 struct dmar_domain *domain;
4598                 u16 did;
4599
4600                 if (!iommu)
4601                         continue;
4602
4603                 for (did = 0; did < 0xffff; did++) {
4604                         domain = get_iommu_domain(iommu, did);
4605
4606                         if (!domain)
4607                                 continue;
4608                         free_cpu_cached_iovas(cpu, &domain->iovad);
4609                 }
4610         }
4611 }
4612
4613 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4614                                     unsigned long action, void *v)
4615 {
4616         unsigned int cpu = (unsigned long)v;
4617
4618         switch (action) {
4619         case CPU_DEAD:
4620         case CPU_DEAD_FROZEN:
4621                 free_all_cpu_cached_iovas(cpu);
4622                 flush_unmaps_timeout(cpu);
4623                 break;
4624         }
4625         return NOTIFY_OK;
4626 }
4627
4628 static struct notifier_block intel_iommu_cpu_nb = {
4629         .notifier_call = intel_iommu_cpu_notifier,
4630 };
4631
4632 static ssize_t intel_iommu_show_version(struct device *dev,
4633                                         struct device_attribute *attr,
4634                                         char *buf)
4635 {
4636         struct intel_iommu *iommu = dev_get_drvdata(dev);
4637         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4638         return sprintf(buf, "%d:%d\n",
4639                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4640 }
4641 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4642
4643 static ssize_t intel_iommu_show_address(struct device *dev,
4644                                         struct device_attribute *attr,
4645                                         char *buf)
4646 {
4647         struct intel_iommu *iommu = dev_get_drvdata(dev);
4648         return sprintf(buf, "%llx\n", iommu->reg_phys);
4649 }
4650 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4651
4652 static ssize_t intel_iommu_show_cap(struct device *dev,
4653                                     struct device_attribute *attr,
4654                                     char *buf)
4655 {
4656         struct intel_iommu *iommu = dev_get_drvdata(dev);
4657         return sprintf(buf, "%llx\n", iommu->cap);
4658 }
4659 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4660
4661 static ssize_t intel_iommu_show_ecap(struct device *dev,
4662                                     struct device_attribute *attr,
4663                                     char *buf)
4664 {
4665         struct intel_iommu *iommu = dev_get_drvdata(dev);
4666         return sprintf(buf, "%llx\n", iommu->ecap);
4667 }
4668 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4669
4670 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4671                                       struct device_attribute *attr,
4672                                       char *buf)
4673 {
4674         struct intel_iommu *iommu = dev_get_drvdata(dev);
4675         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4676 }
4677 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4678
4679 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4680                                            struct device_attribute *attr,
4681                                            char *buf)
4682 {
4683         struct intel_iommu *iommu = dev_get_drvdata(dev);
4684         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4685                                                   cap_ndoms(iommu->cap)));
4686 }
4687 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4688
4689 static struct attribute *intel_iommu_attrs[] = {
4690         &dev_attr_version.attr,
4691         &dev_attr_address.attr,
4692         &dev_attr_cap.attr,
4693         &dev_attr_ecap.attr,
4694         &dev_attr_domains_supported.attr,
4695         &dev_attr_domains_used.attr,
4696         NULL,
4697 };
4698
4699 static struct attribute_group intel_iommu_group = {
4700         .name = "intel-iommu",
4701         .attrs = intel_iommu_attrs,
4702 };
4703
4704 const struct attribute_group *intel_iommu_groups[] = {
4705         &intel_iommu_group,
4706         NULL,
4707 };
4708
4709 int __init intel_iommu_init(void)
4710 {
4711         int ret = -ENODEV;
4712         struct dmar_drhd_unit *drhd;
4713         struct intel_iommu *iommu;
4714
4715         /* VT-d is required for a TXT/tboot launch, so enforce that */
4716         force_on = tboot_force_iommu();
4717
4718         if (iommu_init_mempool()) {
4719                 if (force_on)
4720                         panic("tboot: Failed to initialize iommu memory\n");
4721                 return -ENOMEM;
4722         }
4723
4724         down_write(&dmar_global_lock);
4725         if (dmar_table_init()) {
4726                 if (force_on)
4727                         panic("tboot: Failed to initialize DMAR table\n");
4728                 goto out_free_dmar;
4729         }
4730
4731         if (dmar_dev_scope_init() < 0) {
4732                 if (force_on)
4733                         panic("tboot: Failed to initialize DMAR device scope\n");
4734                 goto out_free_dmar;
4735         }
4736
4737         if (no_iommu || dmar_disabled)
4738                 goto out_free_dmar;
4739
4740         if (list_empty(&dmar_rmrr_units))
4741                 pr_info("No RMRR found\n");
4742
4743         if (list_empty(&dmar_atsr_units))
4744                 pr_info("No ATSR found\n");
4745
4746         if (dmar_init_reserved_ranges()) {
4747                 if (force_on)
4748                         panic("tboot: Failed to reserve iommu ranges\n");
4749                 goto out_free_reserved_range;
4750         }
4751
4752         init_no_remapping_devices();
4753
4754         ret = init_dmars();
4755         if (ret) {
4756                 if (force_on)
4757                         panic("tboot: Failed to initialize DMARs\n");
4758                 pr_err("Initialization failed\n");
4759                 goto out_free_reserved_range;
4760         }
4761         up_write(&dmar_global_lock);
4762         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4763
4764 #ifdef CONFIG_SWIOTLB
4765         swiotlb = 0;
4766 #endif
4767         dma_ops = &intel_dma_ops;
4768
4769         init_iommu_pm_ops();
4770
4771         for_each_active_iommu(iommu, drhd)
4772                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4773                                                        intel_iommu_groups,
4774                                                        "%s", iommu->name);
4775
4776         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4777         bus_register_notifier(&pci_bus_type, &device_nb);
4778         if (si_domain && !hw_pass_through)
4779                 register_memory_notifier(&intel_iommu_memory_nb);
4780         register_hotcpu_notifier(&intel_iommu_cpu_nb);
4781
4782         intel_iommu_enabled = 1;
4783
4784         return 0;
4785
4786 out_free_reserved_range:
4787         put_iova_domain(&reserved_iova_list);
4788 out_free_dmar:
4789         intel_iommu_free_dmars();
4790         up_write(&dmar_global_lock);
4791         iommu_exit_mempool();
4792         return ret;
4793 }
4794
4795 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4796 {
4797         struct intel_iommu *iommu = opaque;
4798
4799         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4800         return 0;
4801 }
4802
4803 /*
4804  * NB - intel-iommu lacks any sort of reference counting for the users of
4805  * dependent devices.  If multiple endpoints have intersecting dependent
4806  * devices, unbinding the driver from any one of them will possibly leave
4807  * the others unable to operate.
4808  */
4809 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4810 {
4811         if (!iommu || !dev || !dev_is_pci(dev))
4812                 return;
4813
4814         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4815 }
4816
4817 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4818 {
4819         struct intel_iommu *iommu;
4820         unsigned long flags;
4821
4822         assert_spin_locked(&device_domain_lock);
4823
4824         if (WARN_ON(!info))
4825                 return;
4826
4827         iommu = info->iommu;
4828
4829         if (info->dev) {
4830                 iommu_disable_dev_iotlb(info);
4831                 domain_context_clear(iommu, info->dev);
4832         }
4833
4834         unlink_domain_info(info);
4835
4836         spin_lock_irqsave(&iommu->lock, flags);
4837         domain_detach_iommu(info->domain, iommu);
4838         spin_unlock_irqrestore(&iommu->lock, flags);
4839
4840         free_devinfo_mem(info);
4841 }
4842
4843 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4844                                      struct device *dev)
4845 {
4846         struct device_domain_info *info;
4847         unsigned long flags;
4848
4849         spin_lock_irqsave(&device_domain_lock, flags);
4850         info = dev->archdata.iommu;
4851         __dmar_remove_one_dev_info(info);
4852         spin_unlock_irqrestore(&device_domain_lock, flags);
4853 }
4854
4855 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4856 {
4857         int adjust_width;
4858
4859         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4860                         DMA_32BIT_PFN);
4861         domain_reserve_special_ranges(domain);
4862
4863         /* calculate AGAW */
4864         domain->gaw = guest_width;
4865         adjust_width = guestwidth_to_adjustwidth(guest_width);
4866         domain->agaw = width_to_agaw(adjust_width);
4867
4868         domain->iommu_coherency = 0;
4869         domain->iommu_snooping = 0;
4870         domain->iommu_superpage = 0;
4871         domain->max_addr = 0;
4872
4873         /* always allocate the top pgd */
4874         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4875         if (!domain->pgd)
4876                 return -ENOMEM;
4877         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4878         return 0;
4879 }
4880
4881 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4882 {
4883         struct dmar_domain *dmar_domain;
4884         struct iommu_domain *domain;
4885
4886         if (type != IOMMU_DOMAIN_UNMANAGED)
4887                 return NULL;
4888
4889         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4890         if (!dmar_domain) {
4891                 pr_err("Can't allocate dmar_domain\n");
4892                 return NULL;
4893         }
4894         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4895                 pr_err("Domain initialization failed\n");
4896                 domain_exit(dmar_domain);
4897                 return NULL;
4898         }
4899         domain_update_iommu_cap(dmar_domain);
4900
4901         domain = &dmar_domain->domain;
4902         domain->geometry.aperture_start = 0;
4903         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4904         domain->geometry.force_aperture = true;
4905
4906         return domain;
4907 }
4908
4909 static void intel_iommu_domain_free(struct iommu_domain *domain)
4910 {
4911         domain_exit(to_dmar_domain(domain));
4912 }
4913
4914 static int intel_iommu_attach_device(struct iommu_domain *domain,
4915                                      struct device *dev)
4916 {
4917         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4918         struct intel_iommu *iommu;
4919         int addr_width;
4920         u8 bus, devfn;
4921
4922         if (device_is_rmrr_locked(dev)) {
4923                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
4924                 return -EPERM;
4925         }
4926
4927         /* normally dev is not mapped */
4928         if (unlikely(domain_context_mapped(dev))) {
4929                 struct dmar_domain *old_domain;
4930
4931                 old_domain = find_domain(dev);
4932                 if (old_domain) {
4933                         rcu_read_lock();
4934                         dmar_remove_one_dev_info(old_domain, dev);
4935                         rcu_read_unlock();
4936
4937                         if (!domain_type_is_vm_or_si(old_domain) &&
4938                              list_empty(&old_domain->devices))
4939                                 domain_exit(old_domain);
4940                 }
4941         }
4942
4943         iommu = device_to_iommu(dev, &bus, &devfn);
4944         if (!iommu)
4945                 return -ENODEV;
4946
4947         /* check if this iommu agaw is sufficient for max mapped address */
4948         addr_width = agaw_to_width(iommu->agaw);
4949         if (addr_width > cap_mgaw(iommu->cap))
4950                 addr_width = cap_mgaw(iommu->cap);
4951
4952         if (dmar_domain->max_addr > (1LL << addr_width)) {
4953                 pr_err("%s: iommu width (%d) is not "
4954                        "sufficient for the mapped address (%llx)\n",
4955                        __func__, addr_width, dmar_domain->max_addr);
4956                 return -EFAULT;
4957         }
4958         dmar_domain->gaw = addr_width;
4959
4960         /*
4961          * Knock out extra levels of page tables if necessary
4962          */
4963         while (iommu->agaw < dmar_domain->agaw) {
4964                 struct dma_pte *pte;
4965
4966                 pte = dmar_domain->pgd;
4967                 if (dma_pte_present(pte)) {
4968                         dmar_domain->pgd = (struct dma_pte *)
4969                                 phys_to_virt(dma_pte_addr(pte));
4970                         free_pgtable_page(pte);
4971                 }
4972                 dmar_domain->agaw--;
4973         }
4974
4975         return domain_add_dev_info(dmar_domain, dev);
4976 }
4977
4978 static void intel_iommu_detach_device(struct iommu_domain *domain,
4979                                       struct device *dev)
4980 {
4981         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4982 }
4983
4984 static int intel_iommu_map(struct iommu_domain *domain,
4985                            unsigned long iova, phys_addr_t hpa,
4986                            size_t size, int iommu_prot)
4987 {
4988         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4989         u64 max_addr;
4990         int prot = 0;
4991         int ret;
4992
4993         if (iommu_prot & IOMMU_READ)
4994                 prot |= DMA_PTE_READ;
4995         if (iommu_prot & IOMMU_WRITE)
4996                 prot |= DMA_PTE_WRITE;
4997         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4998                 prot |= DMA_PTE_SNP;
4999
5000         max_addr = iova + size;
5001         if (dmar_domain->max_addr < max_addr) {
5002                 u64 end;
5003
5004                 /* check if minimum agaw is sufficient for mapped address */
5005                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5006                 if (end < max_addr) {
5007                         pr_err("%s: iommu width (%d) is not "
5008                                "sufficient for the mapped address (%llx)\n",
5009                                __func__, dmar_domain->gaw, max_addr);
5010                         return -EFAULT;
5011                 }
5012                 dmar_domain->max_addr = max_addr;
5013         }
5014         /* Round up size to next multiple of PAGE_SIZE, if it and
5015            the low bits of hpa would take us onto the next page */
5016         size = aligned_nrpages(hpa, size);
5017         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5018                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5019         return ret;
5020 }
5021
5022 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5023                                 unsigned long iova, size_t size)
5024 {
5025         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5026         struct page *freelist = NULL;
5027         struct intel_iommu *iommu;
5028         unsigned long start_pfn, last_pfn;
5029         unsigned int npages;
5030         int iommu_id, level = 0;
5031
5032         /* Cope with horrid API which requires us to unmap more than the
5033            size argument if it happens to be a large-page mapping. */
5034         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5035
5036         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5037                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5038
5039         start_pfn = iova >> VTD_PAGE_SHIFT;
5040         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5041
5042         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5043
5044         npages = last_pfn - start_pfn + 1;
5045
5046         for_each_domain_iommu(iommu_id, dmar_domain) {
5047                 iommu = g_iommus[iommu_id];
5048
5049                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5050                                       start_pfn, npages, !freelist, 0);
5051         }
5052
5053         dma_free_pagelist(freelist);
5054
5055         if (dmar_domain->max_addr == iova + size)
5056                 dmar_domain->max_addr = iova;
5057
5058         return size;
5059 }
5060
5061 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5062                                             dma_addr_t iova)
5063 {
5064         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5065         struct dma_pte *pte;
5066         int level = 0;
5067         u64 phys = 0;
5068
5069         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5070         if (pte)
5071                 phys = dma_pte_addr(pte);
5072
5073         return phys;
5074 }
5075
5076 static bool intel_iommu_capable(enum iommu_cap cap)
5077 {
5078         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5079                 return domain_update_iommu_snooping(NULL) == 1;
5080         if (cap == IOMMU_CAP_INTR_REMAP)
5081                 return irq_remapping_enabled == 1;
5082
5083         return false;
5084 }
5085
5086 static int intel_iommu_add_device(struct device *dev)
5087 {
5088         struct intel_iommu *iommu;
5089         struct iommu_group *group;
5090         u8 bus, devfn;
5091
5092         iommu = device_to_iommu(dev, &bus, &devfn);
5093         if (!iommu)
5094                 return -ENODEV;
5095
5096         iommu_device_link(iommu->iommu_dev, dev);
5097
5098         group = iommu_group_get_for_dev(dev);
5099
5100         if (IS_ERR(group))
5101                 return PTR_ERR(group);
5102
5103         iommu_group_put(group);
5104         return 0;
5105 }
5106
5107 static void intel_iommu_remove_device(struct device *dev)
5108 {
5109         struct intel_iommu *iommu;
5110         u8 bus, devfn;
5111
5112         iommu = device_to_iommu(dev, &bus, &devfn);
5113         if (!iommu)
5114                 return;
5115
5116         iommu_group_remove_device(dev);
5117
5118         iommu_device_unlink(iommu->iommu_dev, dev);
5119 }
5120
5121 #ifdef CONFIG_INTEL_IOMMU_SVM
5122 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5123 {
5124         struct device_domain_info *info;
5125         struct context_entry *context;
5126         struct dmar_domain *domain;
5127         unsigned long flags;
5128         u64 ctx_lo;
5129         int ret;
5130
5131         domain = get_valid_domain_for_dev(sdev->dev);
5132         if (!domain)
5133                 return -EINVAL;
5134
5135         spin_lock_irqsave(&device_domain_lock, flags);
5136         spin_lock(&iommu->lock);
5137
5138         ret = -EINVAL;
5139         info = sdev->dev->archdata.iommu;
5140         if (!info || !info->pasid_supported)
5141                 goto out;
5142
5143         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5144         if (WARN_ON(!context))
5145                 goto out;
5146
5147         ctx_lo = context[0].lo;
5148
5149         sdev->did = domain->iommu_did[iommu->seq_id];
5150         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5151
5152         if (!(ctx_lo & CONTEXT_PASIDE)) {
5153                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5154                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap);
5155                 wmb();
5156                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5157                  * extended to permit requests-with-PASID if the PASIDE bit
5158                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5159                  * however, the PASIDE bit is ignored and requests-with-PASID
5160                  * are unconditionally blocked. Which makes less sense.
5161                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5162                  * "guest mode" translation types depending on whether ATS
5163                  * is available or not. Annoyingly, we can't use the new
5164                  * modes *unless* PASIDE is set. */
5165                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5166                         ctx_lo &= ~CONTEXT_TT_MASK;
5167                         if (info->ats_supported)
5168                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5169                         else
5170                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5171                 }
5172                 ctx_lo |= CONTEXT_PASIDE;
5173                 if (iommu->pasid_state_table)
5174                         ctx_lo |= CONTEXT_DINVE;
5175                 if (info->pri_supported)
5176                         ctx_lo |= CONTEXT_PRS;
5177                 context[0].lo = ctx_lo;
5178                 wmb();
5179                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5180                                            DMA_CCMD_MASK_NOBIT,
5181                                            DMA_CCMD_DEVICE_INVL);
5182         }
5183
5184         /* Enable PASID support in the device, if it wasn't already */
5185         if (!info->pasid_enabled)
5186                 iommu_enable_dev_iotlb(info);
5187
5188         if (info->ats_enabled) {
5189                 sdev->dev_iotlb = 1;
5190                 sdev->qdep = info->ats_qdep;
5191                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5192                         sdev->qdep = 0;
5193         }
5194         ret = 0;
5195
5196  out:
5197         spin_unlock(&iommu->lock);
5198         spin_unlock_irqrestore(&device_domain_lock, flags);
5199
5200         return ret;
5201 }
5202
5203 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5204 {
5205         struct intel_iommu *iommu;
5206         u8 bus, devfn;
5207
5208         if (iommu_dummy(dev)) {
5209                 dev_warn(dev,
5210                          "No IOMMU translation for device; cannot enable SVM\n");
5211                 return NULL;
5212         }
5213
5214         iommu = device_to_iommu(dev, &bus, &devfn);
5215         if ((!iommu)) {
5216                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5217                 return NULL;
5218         }
5219
5220         if (!iommu->pasid_table) {
5221                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5222                 return NULL;
5223         }
5224
5225         return iommu;
5226 }
5227 #endif /* CONFIG_INTEL_IOMMU_SVM */
5228
5229 static const struct iommu_ops intel_iommu_ops = {
5230         .capable        = intel_iommu_capable,
5231         .domain_alloc   = intel_iommu_domain_alloc,
5232         .domain_free    = intel_iommu_domain_free,
5233         .attach_dev     = intel_iommu_attach_device,
5234         .detach_dev     = intel_iommu_detach_device,
5235         .map            = intel_iommu_map,
5236         .unmap          = intel_iommu_unmap,
5237         .map_sg         = default_iommu_map_sg,
5238         .iova_to_phys   = intel_iommu_iova_to_phys,
5239         .add_device     = intel_iommu_add_device,
5240         .remove_device  = intel_iommu_remove_device,
5241         .device_group   = pci_device_group,
5242         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
5243 };
5244
5245 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5246 {
5247         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5248         pr_info("Disabling IOMMU for graphics on this chipset\n");
5249         dmar_map_gfx = 0;
5250 }
5251
5252 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5253 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5254 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5255 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5256 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5257 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5258 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5259
5260 static void quirk_iommu_rwbf(struct pci_dev *dev)
5261 {
5262         /*
5263          * Mobile 4 Series Chipset neglects to set RWBF capability,
5264          * but needs it. Same seems to hold for the desktop versions.
5265          */
5266         pr_info("Forcing write-buffer flush capability\n");
5267         rwbf_quirk = 1;
5268 }
5269
5270 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5271 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5272 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5273 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5274 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5275 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5276 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5277
5278 #define GGC 0x52
5279 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5280 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5281 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5282 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5283 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5284 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5285 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5286 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5287
5288 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5289 {
5290         unsigned short ggc;
5291
5292         if (pci_read_config_word(dev, GGC, &ggc))
5293                 return;
5294
5295         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5296                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5297                 dmar_map_gfx = 0;
5298         } else if (dmar_map_gfx) {
5299                 /* we have to ensure the gfx device is idle before we flush */
5300                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5301                 intel_iommu_strict = 1;
5302        }
5303 }
5304 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5305 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5306 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5307 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5308
5309 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5310    ISOCH DMAR unit for the Azalia sound device, but not give it any
5311    TLB entries, which causes it to deadlock. Check for that.  We do
5312    this in a function called from init_dmars(), instead of in a PCI
5313    quirk, because we don't want to print the obnoxious "BIOS broken"
5314    message if VT-d is actually disabled.
5315 */
5316 static void __init check_tylersburg_isoch(void)
5317 {
5318         struct pci_dev *pdev;
5319         uint32_t vtisochctrl;
5320
5321         /* If there's no Azalia in the system anyway, forget it. */
5322         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5323         if (!pdev)
5324                 return;
5325         pci_dev_put(pdev);
5326
5327         /* System Management Registers. Might be hidden, in which case
5328            we can't do the sanity check. But that's OK, because the
5329            known-broken BIOSes _don't_ actually hide it, so far. */
5330         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5331         if (!pdev)
5332                 return;
5333
5334         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5335                 pci_dev_put(pdev);
5336                 return;
5337         }
5338
5339         pci_dev_put(pdev);
5340
5341         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5342         if (vtisochctrl & 1)
5343                 return;
5344
5345         /* Drop all bits other than the number of TLB entries */
5346         vtisochctrl &= 0x1c;
5347
5348         /* If we have the recommended number of TLB entries (16), fine. */
5349         if (vtisochctrl == 0x10)
5350                 return;
5351
5352         /* Zero TLB entries? You get to ride the short bus to school. */
5353         if (!vtisochctrl) {
5354                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5355                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5356                      dmi_get_system_info(DMI_BIOS_VENDOR),
5357                      dmi_get_system_info(DMI_BIOS_VERSION),
5358                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5359                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5360                 return;
5361         }
5362
5363         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5364                vtisochctrl);
5365 }