drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <linux/syscore_ops.h>
  40 #include <linux/tboot.h>
  41 #include <linux/dmi.h>
  42 #include <linux/pci-ats.h>
  43 #include <asm/cacheflush.h>
  44 #include <asm/iommu.h>
  45
  46 #define ROOT_SIZE               VTD_PAGE_SIZE
  47 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  48
  49 #define IS_BRIDGE_HOST_DEVICE(pdev) \
  50                             ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
  51 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  52 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  53 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  54
  55 #define IOAPIC_RANGE_START      (0xfee00000)
  56 #define IOAPIC_RANGE_END        (0xfeefffff)
  57 #define IOVA_START_ADDR         (0x1000)
  58
  59 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  60
  61 #define MAX_AGAW_WIDTH 64
  62
  63 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  64 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  65
  66 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  67    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  68 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  69                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  70 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  71
  72 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  73 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  74 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  75
  76 /* page table handling */
  77 #define LEVEL_STRIDE            (9)
  78 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  79
  80 static inline int agaw_to_level(int agaw)
  81 {
  82         return agaw + 2;
  83 }
  84
  85 static inline int agaw_to_width(int agaw)
  86 {
  87         return 30 + agaw * LEVEL_STRIDE;
  88 }
  89
  90 static inline int width_to_agaw(int width)
  91 {
  92         return (width - 30) / LEVEL_STRIDE;
  93 }
  94
  95 static inline unsigned int level_to_offset_bits(int level)
  96 {
  97         return (level - 1) * LEVEL_STRIDE;
  98 }
  99
 100 static inline int pfn_level_offset(unsigned long pfn, int level)
 101 {
 102         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 103 }
 104
 105 static inline unsigned long level_mask(int level)
 106 {
 107         return -1UL << level_to_offset_bits(level);
 108 }
 109
 110 static inline unsigned long level_size(int level)
 111 {
 112         return 1UL << level_to_offset_bits(level);
 113 }
 114
 115 static inline unsigned long align_to_level(unsigned long pfn, int level)
 116 {
 117         return (pfn + level_size(level) - 1) & level_mask(level);
 118 }
 119
 120 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 121 {
 122         return  1 << ((lvl - 1) * LEVEL_STRIDE);
 123 }
 124
 125 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 126    are never going to work. */
 127 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 128 {
 129         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 130 }
 131
 132 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 133 {
 134         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 135 }
 136 static inline unsigned long page_to_dma_pfn(struct page *pg)
 137 {
 138         return mm_to_dma_pfn(page_to_pfn(pg));
 139 }
 140 static inline unsigned long virt_to_dma_pfn(void *p)
 141 {
 142         return page_to_dma_pfn(virt_to_page(p));
 143 }
 144
 145 /* global iommu list, set NULL for ignored DMAR units */
 146 static struct intel_iommu **g_iommus;
 147
 148 static void __init check_tylersburg_isoch(void);
 149 static int rwbf_quirk;
 150
 151 /*
 152  * set to 1 to panic kernel if can't successfully enable VT-d
 153  * (used when kernel is launched w/ TXT)
 154  */
 155 static int force_on = 0;
 156
 157 /*
 158  * 0: Present
 159  * 1-11: Reserved
 160  * 12-63: Context Ptr (12 - (haw-1))
 161  * 64-127: Reserved
 162  */
 163 struct root_entry {
 164         u64     val;
 165         u64     rsvd1;
 166 };
 167 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 168 static inline bool root_present(struct root_entry *root)
 169 {
 170         return (root->val & 1);
 171 }
 172 static inline void set_root_present(struct root_entry *root)
 173 {
 174         root->val |= 1;
 175 }
 176 static inline void set_root_value(struct root_entry *root, unsigned long value)
 177 {
 178         root->val |= value & VTD_PAGE_MASK;
 179 }
 180
 181 static inline struct context_entry *
 182 get_context_addr_from_root(struct root_entry *root)
 183 {
 184         return (struct context_entry *)
 185                 (root_present(root)?phys_to_virt(
 186                 root->val & VTD_PAGE_MASK) :
 187                 NULL);
 188 }
 189
 190 /*
 191  * low 64 bits:
 192  * 0: present
 193  * 1: fault processing disable
 194  * 2-3: translation type
 195  * 12-63: address space root
 196  * high 64 bits:
 197  * 0-2: address width
 198  * 3-6: aval
 199  * 8-23: domain id
 200  */
 201 struct context_entry {
 202         u64 lo;
 203         u64 hi;
 204 };
 205
 206 static inline bool context_present(struct context_entry *context)
 207 {
 208         return (context->lo & 1);
 209 }
 210 static inline void context_set_present(struct context_entry *context)
 211 {
 212         context->lo |= 1;
 213 }
 214
 215 static inline void context_set_fault_enable(struct context_entry *context)
 216 {
 217         context->lo &= (((u64)-1) << 2) | 1;
 218 }
 219
 220 static inline void context_set_translation_type(struct context_entry *context,
 221                                                 unsigned long value)
 222 {
 223         context->lo &= (((u64)-1) << 4) | 3;
 224         context->lo |= (value & 3) << 2;
 225 }
 226
 227 static inline void context_set_address_root(struct context_entry *context,
 228                                             unsigned long value)
 229 {
 230         context->lo |= value & VTD_PAGE_MASK;
 231 }
 232
 233 static inline void context_set_address_width(struct context_entry *context,
 234                                              unsigned long value)
 235 {
 236         context->hi |= value & 7;
 237 }
 238
 239 static inline void context_set_domain_id(struct context_entry *context,
 240                                          unsigned long value)
 241 {
 242         context->hi |= (value & ((1 << 16) - 1)) << 8;
 243 }
 244
 245 static inline void context_clear_entry(struct context_entry *context)
 246 {
 247         context->lo = 0;
 248         context->hi = 0;
 249 }
 250
 251 /*
 252  * 0: readable
 253  * 1: writable
 254  * 2-6: reserved
 255  * 7: super page
 256  * 8-10: available
 257  * 11: snoop behavior
 258  * 12-63: Host physcial address
 259  */
 260 struct dma_pte {
 261         u64 val;
 262 };
 263
 264 static inline void dma_clear_pte(struct dma_pte *pte)
 265 {
 266         pte->val = 0;
 267 }
 268
 269 static inline void dma_set_pte_readable(struct dma_pte *pte)
 270 {
 271         pte->val |= DMA_PTE_READ;
 272 }
 273
 274 static inline void dma_set_pte_writable(struct dma_pte *pte)
 275 {
 276         pte->val |= DMA_PTE_WRITE;
 277 }
 278
 279 static inline void dma_set_pte_snp(struct dma_pte *pte)
 280 {
 281         pte->val |= DMA_PTE_SNP;
 282 }
 283
 284 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 285 {
 286         pte->val = (pte->val & ~3) | (prot & 3);
 287 }
 288
 289 static inline u64 dma_pte_addr(struct dma_pte *pte)
 290 {
 291 #ifdef CONFIG_64BIT
 292         return pte->val & VTD_PAGE_MASK;
 293 #else
 294         /* Must have a full atomic 64-bit read */
 295         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 296 #endif
 297 }
 298
 299 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 300 {
 301         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 302 }
 303
 304 static inline bool dma_pte_present(struct dma_pte *pte)
 305 {
 306         return (pte->val & 3) != 0;
 307 }
 308
 309 static inline int first_pte_in_page(struct dma_pte *pte)
 310 {
 311         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 312 }
 313
 314 /*
 315  * This domain is a statically identity mapping domain.
 316  *      1. This domain creats a static 1:1 mapping to all usable memory.
 317  *      2. It maps to each iommu if successful.
 318  *      3. Each iommu mapps to this domain if successful.
 319  */
 320 static struct dmar_domain *si_domain;
 321 static int hw_pass_through = 1;
 322
 323 /* devices under the same p2p bridge are owned in one domain */
 324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 325
 326 /* domain represents a virtual machine, more than one devices
 327  * across iommus may be owned in one domain, e.g. kvm guest.
 328  */
 329 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 330
 331 /* si_domain contains mulitple devices */
 332 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 333
 334 struct dmar_domain {
 335         int     id;                     /* domain id */
 336         int     nid;                    /* node id */
 337         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 338
 339         struct list_head devices;       /* all devices' list */
 340         struct iova_domain iovad;       /* iova's that belong to this domain */
 341
 342         struct dma_pte  *pgd;           /* virtual address */
 343         int             gaw;            /* max guest address width */
 344
 345         /* adjusted guest address width, 0 is level 2 30-bit */
 346         int             agaw;
 347
 348         int             flags;          /* flags to find out type of domain */
 349
 350         int             iommu_coherency;/* indicate coherency of iommu access */
 351         int             iommu_snooping; /* indicate snooping control feature*/
 352         int             iommu_count;    /* reference count of iommu */
 353         int             iommu_superpage;/* Level of superpages supported:
 354                                            0 == 4KiB (no superpages), 1 == 2MiB,
 355                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 356         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 357         u64             max_addr;       /* maximum mapped address */
 358 };
 359
 360 /* PCI domain-device relationship */
 361 struct device_domain_info {
 362         struct list_head link;  /* link to domain siblings */
 363         struct list_head global; /* link to global list */
 364         int segment;            /* PCI domain */
 365         u8 bus;                 /* PCI bus number */
 366         u8 devfn;               /* PCI devfn number */
 367         struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 368         struct intel_iommu *iommu; /* IOMMU used by this device */
 369         struct dmar_domain *domain; /* pointer to domain */
 370 };
 371
 372 static void flush_unmaps_timeout(unsigned long data);
 373
 374 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 375
 376 #define HIGH_WATER_MARK 250
 377 struct deferred_flush_tables {
 378         int next;
 379         struct iova *iova[HIGH_WATER_MARK];
 380         struct dmar_domain *domain[HIGH_WATER_MARK];
 381 };
 382
 383 static struct deferred_flush_tables *deferred_flush;
 384
 385 /* bitmap for indexing intel_iommus */
 386 static int g_num_of_iommus;
 387
 388 static DEFINE_SPINLOCK(async_umap_flush_lock);
 389 static LIST_HEAD(unmaps_to_do);
 390
 391 static int timer_on;
 392 static long list_size;
 393
 394 static void domain_remove_dev_info(struct dmar_domain *domain);
 395
 396 #ifdef CONFIG_DMAR_DEFAULT_ON
 397 int dmar_disabled = 0;
 398 #else
 399 int dmar_disabled = 1;
 400 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 401
 402 static int dmar_map_gfx = 1;
 403 static int dmar_forcedac;
 404 static int intel_iommu_strict;
 405 static int intel_iommu_superpage = 1;
 406
 407 int intel_iommu_gfx_mapped;
 408 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 409
 410 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 411 static DEFINE_SPINLOCK(device_domain_lock);
 412 static LIST_HEAD(device_domain_list);
 413
 414 static struct iommu_ops intel_iommu_ops;
 415
 416 static int __init intel_iommu_setup(char *str)
 417 {
 418         if (!str)
 419                 return -EINVAL;
 420         while (*str) {
 421                 if (!strncmp(str, "on", 2)) {
 422                         dmar_disabled = 0;
 423                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 424                 } else if (!strncmp(str, "off", 3)) {
 425                         dmar_disabled = 1;
 426                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 427                 } else if (!strncmp(str, "igfx_off", 8)) {
 428                         dmar_map_gfx = 0;
 429                         printk(KERN_INFO
 430                                 "Intel-IOMMU: disable GFX device mapping\n");
 431                 } else if (!strncmp(str, "forcedac", 8)) {
 432                         printk(KERN_INFO
 433                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 434                         dmar_forcedac = 1;
 435                 } else if (!strncmp(str, "strict", 6)) {
 436                         printk(KERN_INFO
 437                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 438                         intel_iommu_strict = 1;
 439                 } else if (!strncmp(str, "sp_off", 6)) {
 440                         printk(KERN_INFO
 441                                 "Intel-IOMMU: disable supported super page\n");
 442                         intel_iommu_superpage = 0;
 443                 }
 444
 445                 str += strcspn(str, ",");
 446                 while (*str == ',')
 447                         str++;
 448         }
 449         return 0;
 450 }
 451 __setup("intel_iommu=", intel_iommu_setup);
 452
 453 static struct kmem_cache *iommu_domain_cache;
 454 static struct kmem_cache *iommu_devinfo_cache;
 455 static struct kmem_cache *iommu_iova_cache;
 456
 457 static inline void *alloc_pgtable_page(int node)
 458 {
 459         struct page *page;
 460         void *vaddr = NULL;
 461
 462         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 463         if (page)
 464                 vaddr = page_address(page);
 465         return vaddr;
 466 }
 467
 468 static inline void free_pgtable_page(void *vaddr)
 469 {
 470         free_page((unsigned long)vaddr);
 471 }
 472
 473 static inline void *alloc_domain_mem(void)
 474 {
 475         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 476 }
 477
 478 static void free_domain_mem(void *vaddr)
 479 {
 480         kmem_cache_free(iommu_domain_cache, vaddr);
 481 }
 482
 483 static inline void * alloc_devinfo_mem(void)
 484 {
 485         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 486 }
 487
 488 static inline void free_devinfo_mem(void *vaddr)
 489 {
 490         kmem_cache_free(iommu_devinfo_cache, vaddr);
 491 }
 492
 493 struct iova *alloc_iova_mem(void)
 494 {
 495         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 496 }
 497
 498 void free_iova_mem(struct iova *iova)
 499 {
 500         kmem_cache_free(iommu_iova_cache, iova);
 501 }
 502
 503
 504 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 505 {
 506         unsigned long sagaw;
 507         int agaw = -1;
 508
 509         sagaw = cap_sagaw(iommu->cap);
 510         for (agaw = width_to_agaw(max_gaw);
 511              agaw >= 0; agaw--) {
 512                 if (test_bit(agaw, &sagaw))
 513                         break;
 514         }
 515
 516         return agaw;
 517 }
 518
 519 /*
 520  * Calculate max SAGAW for each iommu.
 521  */
 522 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 523 {
 524         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 525 }
 526
 527 /*
 528  * calculate agaw for each iommu.
 529  * "SAGAW" may be different across iommus, use a default agaw, and
 530  * get a supported less agaw for iommus that don't support the default agaw.
 531  */
 532 int iommu_calculate_agaw(struct intel_iommu *iommu)
 533 {
 534         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 535 }
 536
 537 /* This functionin only returns single iommu in a domain */
 538 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 539 {
 540         int iommu_id;
 541
 542         /* si_domain and vm domain should not get here. */
 543         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 544         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 545
 546         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 547         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 548                 return NULL;
 549
 550         return g_iommus[iommu_id];
 551 }
 552
 553 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 554 {
 555         int i;
 556
 557         domain->iommu_coherency = 1;
 558
 559         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 560                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 561                         domain->iommu_coherency = 0;
 562                         break;
 563                 }
 564         }
 565 }
 566
 567 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 568 {
 569         int i;
 570
 571         domain->iommu_snooping = 1;
 572
 573         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 574                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 575                         domain->iommu_snooping = 0;
 576                         break;
 577                 }
 578         }
 579 }
 580
 581 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 582 {
 583         struct dmar_drhd_unit *drhd;
 584         struct intel_iommu *iommu = NULL;
 585         int mask = 0xf;
 586
 587         if (!intel_iommu_superpage) {
 588                 domain->iommu_superpage = 0;
 589                 return;
 590         }
 591
 592         /* set iommu_superpage to the smallest common denominator */
 593         for_each_active_iommu(iommu, drhd) {
 594                 mask &= cap_super_page_val(iommu->cap);
 595                 if (!mask) {
 596                         break;
 597                 }
 598         }
 599         domain->iommu_superpage = fls(mask);
 600 }
 601
 602 /* Some capabilities may be different across iommus */
 603 static void domain_update_iommu_cap(struct dmar_domain *domain)
 604 {
 605         domain_update_iommu_coherency(domain);
 606         domain_update_iommu_snooping(domain);
 607         domain_update_iommu_superpage(domain);
 608 }
 609
 610 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 611 {
 612         struct dmar_drhd_unit *drhd = NULL;
 613         int i;
 614
 615         for_each_drhd_unit(drhd) {
 616                 if (drhd->ignored)
 617                         continue;
 618                 if (segment != drhd->segment)
 619                         continue;
 620
 621                 for (i = 0; i < drhd->devices_cnt; i++) {
 622                         if (drhd->devices[i] &&
 623                             drhd->devices[i]->bus->number == bus &&
 624                             drhd->devices[i]->devfn == devfn)
 625                                 return drhd->iommu;
 626                         if (drhd->devices[i] &&
 627                             drhd->devices[i]->subordinate &&
 628                             drhd->devices[i]->subordinate->number <= bus &&
 629                             drhd->devices[i]->subordinate->subordinate >= bus)
 630                                 return drhd->iommu;
 631                 }
 632
 633                 if (drhd->include_all)
 634                         return drhd->iommu;
 635         }
 636
 637         return NULL;
 638 }
 639
 640 static void domain_flush_cache(struct dmar_domain *domain,
 641                                void *addr, int size)
 642 {
 643         if (!domain->iommu_coherency)
 644                 clflush_cache_range(addr, size);
 645 }
 646
 647 /* Gets context entry for a given bus and devfn */
 648 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 649                 u8 bus, u8 devfn)
 650 {
 651         struct root_entry *root;
 652         struct context_entry *context;
 653         unsigned long phy_addr;
 654         unsigned long flags;
 655
 656         spin_lock_irqsave(&iommu->lock, flags);
 657         root = &iommu->root_entry[bus];
 658         context = get_context_addr_from_root(root);
 659         if (!context) {
 660                 context = (struct context_entry *)
 661                                 alloc_pgtable_page(iommu->node);
 662                 if (!context) {
 663                         spin_unlock_irqrestore(&iommu->lock, flags);
 664                         return NULL;
 665                 }
 666                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 667                 phy_addr = virt_to_phys((void *)context);
 668                 set_root_value(root, phy_addr);
 669                 set_root_present(root);
 670                 __iommu_flush_cache(iommu, root, sizeof(*root));
 671         }
 672         spin_unlock_irqrestore(&iommu->lock, flags);
 673         return &context[devfn];
 674 }
 675
 676 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 677 {
 678         struct root_entry *root;
 679         struct context_entry *context;
 680         int ret;
 681         unsigned long flags;
 682
 683         spin_lock_irqsave(&iommu->lock, flags);
 684         root = &iommu->root_entry[bus];
 685         context = get_context_addr_from_root(root);
 686         if (!context) {
 687                 ret = 0;
 688                 goto out;
 689         }
 690         ret = context_present(&context[devfn]);
 691 out:
 692         spin_unlock_irqrestore(&iommu->lock, flags);
 693         return ret;
 694 }
 695
 696 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 697 {
 698         struct root_entry *root;
 699         struct context_entry *context;
 700         unsigned long flags;
 701
 702         spin_lock_irqsave(&iommu->lock, flags);
 703         root = &iommu->root_entry[bus];
 704         context = get_context_addr_from_root(root);
 705         if (context) {
 706                 context_clear_entry(&context[devfn]);
 707                 __iommu_flush_cache(iommu, &context[devfn], \
 708                         sizeof(*context));
 709         }
 710         spin_unlock_irqrestore(&iommu->lock, flags);
 711 }
 712
 713 static void free_context_table(struct intel_iommu *iommu)
 714 {
 715         struct root_entry *root;
 716         int i;
 717         unsigned long flags;
 718         struct context_entry *context;
 719
 720         spin_lock_irqsave(&iommu->lock, flags);
 721         if (!iommu->root_entry) {
 722                 goto out;
 723         }
 724         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 725                 root = &iommu->root_entry[i];
 726                 context = get_context_addr_from_root(root);
 727                 if (context)
 728                         free_pgtable_page(context);
 729         }
 730         free_pgtable_page(iommu->root_entry);
 731         iommu->root_entry = NULL;
 732 out:
 733         spin_unlock_irqrestore(&iommu->lock, flags);
 734 }
 735
 736 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 737                                       unsigned long pfn, int large_level)
 738 {
 739         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 740         struct dma_pte *parent, *pte = NULL;
 741         int level = agaw_to_level(domain->agaw);
 742         int offset, target_level;
 743
 744         BUG_ON(!domain->pgd);
 745         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 746         parent = domain->pgd;
 747
 748         /* Search pte */
 749         if (!large_level)
 750                 target_level = 1;
 751         else
 752                 target_level = large_level;
 753
 754         while (level > 0) {
 755                 void *tmp_page;
 756
 757                 offset = pfn_level_offset(pfn, level);
 758                 pte = &parent[offset];
 759                 if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
 760                         break;
 761                 if (level == target_level)
 762                         break;
 763
 764                 if (!dma_pte_present(pte)) {
 765                         uint64_t pteval;
 766
 767                         tmp_page = alloc_pgtable_page(domain->nid);
 768
 769                         if (!tmp_page)
 770                                 return NULL;
 771
 772                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 773                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 774                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 775                                 /* Someone else set it while we were thinking; use theirs. */
 776                                 free_pgtable_page(tmp_page);
 777                         } else {
 778                                 dma_pte_addr(pte);
 779                                 domain_flush_cache(domain, pte, sizeof(*pte));
 780                         }
 781                 }
 782                 parent = phys_to_virt(dma_pte_addr(pte));
 783                 level--;
 784         }
 785
 786         return pte;
 787 }
 788
 789
 790 /* return address's pte at specific level */
 791 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 792                                          unsigned long pfn,
 793                                          int level, int *large_page)
 794 {
 795         struct dma_pte *parent, *pte = NULL;
 796         int total = agaw_to_level(domain->agaw);
 797         int offset;
 798
 799         parent = domain->pgd;
 800         while (level <= total) {
 801                 offset = pfn_level_offset(pfn, total);
 802                 pte = &parent[offset];
 803                 if (level == total)
 804                         return pte;
 805
 806                 if (!dma_pte_present(pte)) {
 807                         *large_page = total;
 808                         break;
 809                 }
 810
 811                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 812                         *large_page = total;
 813                         return pte;
 814                 }
 815
 816                 parent = phys_to_virt(dma_pte_addr(pte));
 817                 total--;
 818         }
 819         return NULL;
 820 }
 821
 822 /* clear last level pte, a tlb flush should be followed */
 823 static int dma_pte_clear_range(struct dmar_domain *domain,
 824                                 unsigned long start_pfn,
 825                                 unsigned long last_pfn)
 826 {
 827         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 828         unsigned int large_page = 1;
 829         struct dma_pte *first_pte, *pte;
 830         int order;
 831
 832         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 833         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 834         BUG_ON(start_pfn > last_pfn);
 835
 836         /* we don't need lock here; nobody else touches the iova range */
 837         do {
 838                 large_page = 1;
 839                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 840                 if (!pte) {
 841                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 842                         continue;
 843                 }
 844                 do {
 845                         dma_clear_pte(pte);
 846                         start_pfn += lvl_to_nr_pages(large_page);
 847                         pte++;
 848                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 849
 850                 domain_flush_cache(domain, first_pte,
 851                                    (void *)pte - (void *)first_pte);
 852
 853         } while (start_pfn && start_pfn <= last_pfn);
 854
 855         order = (large_page - 1) * 9;
 856         return order;
 857 }
 858
 859 /* free page table pages. last level pte should already be cleared */
 860 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 861                                    unsigned long start_pfn,
 862                                    unsigned long last_pfn)
 863 {
 864         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 865         struct dma_pte *first_pte, *pte;
 866         int total = agaw_to_level(domain->agaw);
 867         int level;
 868         unsigned long tmp;
 869         int large_page = 2;
 870
 871         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 872         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 873         BUG_ON(start_pfn > last_pfn);
 874
 875         /* We don't need lock here; nobody else touches the iova range */
 876         level = 2;
 877         while (level <= total) {
 878                 tmp = align_to_level(start_pfn, level);
 879
 880                 /* If we can't even clear one PTE at this level, we're done */
 881                 if (tmp + level_size(level) - 1 > last_pfn)
 882                         return;
 883
 884                 do {
 885                         large_page = level;
 886                         first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
 887                         if (large_page > level)
 888                                 level = large_page + 1;
 889                         if (!pte) {
 890                                 tmp = align_to_level(tmp + 1, level + 1);
 891                                 continue;
 892                         }
 893                         do {
 894                                 if (dma_pte_present(pte)) {
 895                                         free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 896                                         dma_clear_pte(pte);
 897                                 }
 898                                 pte++;
 899                                 tmp += level_size(level);
 900                         } while (!first_pte_in_page(pte) &&
 901                                  tmp + level_size(level) - 1 <= last_pfn);
 902
 903                         domain_flush_cache(domain, first_pte,
 904                                            (void *)pte - (void *)first_pte);
 905
 906                 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 907                 level++;
 908         }
 909         /* free pgd */
 910         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 911                 free_pgtable_page(domain->pgd);
 912                 domain->pgd = NULL;
 913         }
 914 }
 915
 916 /* iommu handling */
 917 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 918 {
 919         struct root_entry *root;
 920         unsigned long flags;
 921
 922         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 923         if (!root)
 924                 return -ENOMEM;
 925
 926         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 927
 928         spin_lock_irqsave(&iommu->lock, flags);
 929         iommu->root_entry = root;
 930         spin_unlock_irqrestore(&iommu->lock, flags);
 931
 932         return 0;
 933 }
 934
 935 static void iommu_set_root_entry(struct intel_iommu *iommu)
 936 {
 937         void *addr;
 938         u32 sts;
 939         unsigned long flag;
 940
 941         addr = iommu->root_entry;
 942
 943         spin_lock_irqsave(&iommu->register_lock, flag);
 944         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 945
 946         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 947
 948         /* Make sure hardware complete it */
 949         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 950                       readl, (sts & DMA_GSTS_RTPS), sts);
 951
 952         spin_unlock_irqrestore(&iommu->register_lock, flag);
 953 }
 954
 955 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 956 {
 957         u32 val;
 958         unsigned long flag;
 959
 960         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 961                 return;
 962
 963         spin_lock_irqsave(&iommu->register_lock, flag);
 964         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 965
 966         /* Make sure hardware complete it */
 967         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 968                       readl, (!(val & DMA_GSTS_WBFS)), val);
 969
 970         spin_unlock_irqrestore(&iommu->register_lock, flag);
 971 }
 972
 973 /* return value determine if we need a write buffer flush */
 974 static void __iommu_flush_context(struct intel_iommu *iommu,
 975                                   u16 did, u16 source_id, u8 function_mask,
 976                                   u64 type)
 977 {
 978         u64 val = 0;
 979         unsigned long flag;
 980
 981         switch (type) {
 982         case DMA_CCMD_GLOBAL_INVL:
 983                 val = DMA_CCMD_GLOBAL_INVL;
 984                 break;
 985         case DMA_CCMD_DOMAIN_INVL:
 986                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 987                 break;
 988         case DMA_CCMD_DEVICE_INVL:
 989                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 990                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 991                 break;
 992         default:
 993                 BUG();
 994         }
 995         val |= DMA_CCMD_ICC;
 996
 997         spin_lock_irqsave(&iommu->register_lock, flag);
 998         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 999
1000         /* Make sure hardware complete it */
1001         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1002                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1003
1004         spin_unlock_irqrestore(&iommu->register_lock, flag);
1005 }
1006
1007 /* return value determine if we need a write buffer flush */
1008 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1009                                 u64 addr, unsigned int size_order, u64 type)
1010 {
1011         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1012         u64 val = 0, val_iva = 0;
1013         unsigned long flag;
1014
1015         switch (type) {
1016         case DMA_TLB_GLOBAL_FLUSH:
1017                 /* global flush doesn't need set IVA_REG */
1018                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1019                 break;
1020         case DMA_TLB_DSI_FLUSH:
1021                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1022                 break;
1023         case DMA_TLB_PSI_FLUSH:
1024                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1025                 /* Note: always flush non-leaf currently */
1026                 val_iva = size_order | addr;
1027                 break;
1028         default:
1029                 BUG();
1030         }
1031         /* Note: set drain read/write */
1032 #if 0
1033         /*
1034          * This is probably to be super secure.. Looks like we can
1035          * ignore it without any impact.
1036          */
1037         if (cap_read_drain(iommu->cap))
1038                 val |= DMA_TLB_READ_DRAIN;
1039 #endif
1040         if (cap_write_drain(iommu->cap))
1041                 val |= DMA_TLB_WRITE_DRAIN;
1042
1043         spin_lock_irqsave(&iommu->register_lock, flag);
1044         /* Note: Only uses first TLB reg currently */
1045         if (val_iva)
1046                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1047         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1048
1049         /* Make sure hardware complete it */
1050         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1051                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1052
1053         spin_unlock_irqrestore(&iommu->register_lock, flag);
1054
1055         /* check IOTLB invalidation granularity */
1056         if (DMA_TLB_IAIG(val) == 0)
1057                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1058         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1059                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1060                         (unsigned long long)DMA_TLB_IIRG(type),
1061                         (unsigned long long)DMA_TLB_IAIG(val));
1062 }
1063
1064 static struct device_domain_info *iommu_support_dev_iotlb(
1065         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1066 {
1067         int found = 0;
1068         unsigned long flags;
1069         struct device_domain_info *info;
1070         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1071
1072         if (!ecap_dev_iotlb_support(iommu->ecap))
1073                 return NULL;
1074
1075         if (!iommu->qi)
1076                 return NULL;
1077
1078         spin_lock_irqsave(&device_domain_lock, flags);
1079         list_for_each_entry(info, &domain->devices, link)
1080                 if (info->bus == bus && info->devfn == devfn) {
1081                         found = 1;
1082                         break;
1083                 }
1084         spin_unlock_irqrestore(&device_domain_lock, flags);
1085
1086         if (!found || !info->dev)
1087                 return NULL;
1088
1089         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1090                 return NULL;
1091
1092         if (!dmar_find_matched_atsr_unit(info->dev))
1093                 return NULL;
1094
1095         info->iommu = iommu;
1096
1097         return info;
1098 }
1099
1100 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1101 {
1102         if (!info)
1103                 return;
1104
1105         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1106 }
1107
1108 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1109 {
1110         if (!info->dev || !pci_ats_enabled(info->dev))
1111                 return;
1112
1113         pci_disable_ats(info->dev);
1114 }
1115
1116 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1117                                   u64 addr, unsigned mask)
1118 {
1119         u16 sid, qdep;
1120         unsigned long flags;
1121         struct device_domain_info *info;
1122
1123         spin_lock_irqsave(&device_domain_lock, flags);
1124         list_for_each_entry(info, &domain->devices, link) {
1125                 if (!info->dev || !pci_ats_enabled(info->dev))
1126                         continue;
1127
1128                 sid = info->bus << 8 | info->devfn;
1129                 qdep = pci_ats_queue_depth(info->dev);
1130                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1131         }
1132         spin_unlock_irqrestore(&device_domain_lock, flags);
1133 }
1134
1135 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1136                                   unsigned long pfn, unsigned int pages, int map)
1137 {
1138         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1139         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1140
1141         BUG_ON(pages == 0);
1142
1143         /*
1144          * Fallback to domain selective flush if no PSI support or the size is
1145          * too big.
1146          * PSI requires page size to be 2 ^ x, and the base address is naturally
1147          * aligned to the size
1148          */
1149         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1150                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1151                                                 DMA_TLB_DSI_FLUSH);
1152         else
1153                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1154                                                 DMA_TLB_PSI_FLUSH);
1155
1156         /*
1157          * In caching mode, changes of pages from non-present to present require
1158          * flush. However, device IOTLB doesn't need to be flushed in this case.
1159          */
1160         if (!cap_caching_mode(iommu->cap) || !map)
1161                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1162 }
1163
1164 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1165 {
1166         u32 pmen;
1167         unsigned long flags;
1168
1169         spin_lock_irqsave(&iommu->register_lock, flags);
1170         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1171         pmen &= ~DMA_PMEN_EPM;
1172         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1173
1174         /* wait for the protected region status bit to clear */
1175         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1176                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1177
1178         spin_unlock_irqrestore(&iommu->register_lock, flags);
1179 }
1180
1181 static int iommu_enable_translation(struct intel_iommu *iommu)
1182 {
1183         u32 sts;
1184         unsigned long flags;
1185
1186         spin_lock_irqsave(&iommu->register_lock, flags);
1187         iommu->gcmd |= DMA_GCMD_TE;
1188         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1189
1190         /* Make sure hardware complete it */
1191         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1192                       readl, (sts & DMA_GSTS_TES), sts);
1193
1194         spin_unlock_irqrestore(&iommu->register_lock, flags);
1195         return 0;
1196 }
1197
1198 static int iommu_disable_translation(struct intel_iommu *iommu)
1199 {
1200         u32 sts;
1201         unsigned long flag;
1202
1203         spin_lock_irqsave(&iommu->register_lock, flag);
1204         iommu->gcmd &= ~DMA_GCMD_TE;
1205         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1206
1207         /* Make sure hardware complete it */
1208         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1209                       readl, (!(sts & DMA_GSTS_TES)), sts);
1210
1211         spin_unlock_irqrestore(&iommu->register_lock, flag);
1212         return 0;
1213 }
1214
1215
1216 static int iommu_init_domains(struct intel_iommu *iommu)
1217 {
1218         unsigned long ndomains;
1219         unsigned long nlongs;
1220
1221         ndomains = cap_ndoms(iommu->cap);
1222         pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1223                         ndomains);
1224         nlongs = BITS_TO_LONGS(ndomains);
1225
1226         spin_lock_init(&iommu->lock);
1227
1228         /* TBD: there might be 64K domains,
1229          * consider other allocation for future chip
1230          */
1231         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1232         if (!iommu->domain_ids) {
1233                 printk(KERN_ERR "Allocating domain id array failed\n");
1234                 return -ENOMEM;
1235         }
1236         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1237                         GFP_KERNEL);
1238         if (!iommu->domains) {
1239                 printk(KERN_ERR "Allocating domain array failed\n");
1240                 return -ENOMEM;
1241         }
1242
1243         /*
1244          * if Caching mode is set, then invalid translations are tagged
1245          * with domainid 0. Hence we need to pre-allocate it.
1246          */
1247         if (cap_caching_mode(iommu->cap))
1248                 set_bit(0, iommu->domain_ids);
1249         return 0;
1250 }
1251
1252
1253 static void domain_exit(struct dmar_domain *domain);
1254 static void vm_domain_exit(struct dmar_domain *domain);
1255
1256 void free_dmar_iommu(struct intel_iommu *iommu)
1257 {
1258         struct dmar_domain *domain;
1259         int i;
1260         unsigned long flags;
1261
1262         if ((iommu->domains) && (iommu->domain_ids)) {
1263                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1264                         domain = iommu->domains[i];
1265                         clear_bit(i, iommu->domain_ids);
1266
1267                         spin_lock_irqsave(&domain->iommu_lock, flags);
1268                         if (--domain->iommu_count == 0) {
1269                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1270                                         vm_domain_exit(domain);
1271                                 else
1272                                         domain_exit(domain);
1273                         }
1274                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1275                 }
1276         }
1277
1278         if (iommu->gcmd & DMA_GCMD_TE)
1279                 iommu_disable_translation(iommu);
1280
1281         if (iommu->irq) {
1282                 irq_set_handler_data(iommu->irq, NULL);
1283                 /* This will mask the irq */
1284                 free_irq(iommu->irq, iommu);
1285                 destroy_irq(iommu->irq);
1286         }
1287
1288         kfree(iommu->domains);
1289         kfree(iommu->domain_ids);
1290
1291         g_iommus[iommu->seq_id] = NULL;
1292
1293         /* if all iommus are freed, free g_iommus */
1294         for (i = 0; i < g_num_of_iommus; i++) {
1295                 if (g_iommus[i])
1296                         break;
1297         }
1298
1299         if (i == g_num_of_iommus)
1300                 kfree(g_iommus);
1301
1302         /* free context mapping */
1303         free_context_table(iommu);
1304 }
1305
1306 static struct dmar_domain *alloc_domain(void)
1307 {
1308         struct dmar_domain *domain;
1309
1310         domain = alloc_domain_mem();
1311         if (!domain)
1312                 return NULL;
1313
1314         domain->nid = -1;
1315         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1316         domain->flags = 0;
1317
1318         return domain;
1319 }
1320
1321 static int iommu_attach_domain(struct dmar_domain *domain,
1322                                struct intel_iommu *iommu)
1323 {
1324         int num;
1325         unsigned long ndomains;
1326         unsigned long flags;
1327
1328         ndomains = cap_ndoms(iommu->cap);
1329
1330         spin_lock_irqsave(&iommu->lock, flags);
1331
1332         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1333         if (num >= ndomains) {
1334                 spin_unlock_irqrestore(&iommu->lock, flags);
1335                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1336                 return -ENOMEM;
1337         }
1338
1339         domain->id = num;
1340         set_bit(num, iommu->domain_ids);
1341         set_bit(iommu->seq_id, &domain->iommu_bmp);
1342         iommu->domains[num] = domain;
1343         spin_unlock_irqrestore(&iommu->lock, flags);
1344
1345         return 0;
1346 }
1347
1348 static void iommu_detach_domain(struct dmar_domain *domain,
1349                                 struct intel_iommu *iommu)
1350 {
1351         unsigned long flags;
1352         int num, ndomains;
1353         int found = 0;
1354
1355         spin_lock_irqsave(&iommu->lock, flags);
1356         ndomains = cap_ndoms(iommu->cap);
1357         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1358                 if (iommu->domains[num] == domain) {
1359                         found = 1;
1360                         break;
1361                 }
1362         }
1363
1364         if (found) {
1365                 clear_bit(num, iommu->domain_ids);
1366                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1367                 iommu->domains[num] = NULL;
1368         }
1369         spin_unlock_irqrestore(&iommu->lock, flags);
1370 }
1371
1372 static struct iova_domain reserved_iova_list;
1373 static struct lock_class_key reserved_rbtree_key;
1374
1375 static int dmar_init_reserved_ranges(void)
1376 {
1377         struct pci_dev *pdev = NULL;
1378         struct iova *iova;
1379         int i;
1380
1381         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1382
1383         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1384                 &reserved_rbtree_key);
1385
1386         /* IOAPIC ranges shouldn't be accessed by DMA */
1387         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1388                 IOVA_PFN(IOAPIC_RANGE_END));
1389         if (!iova) {
1390                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1391                 return -ENODEV;
1392         }
1393
1394         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1395         for_each_pci_dev(pdev) {
1396                 struct resource *r;
1397
1398                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1399                         r = &pdev->resource[i];
1400                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1401                                 continue;
1402                         iova = reserve_iova(&reserved_iova_list,
1403                                             IOVA_PFN(r->start),
1404                                             IOVA_PFN(r->end));
1405                         if (!iova) {
1406                                 printk(KERN_ERR "Reserve iova failed\n");
1407                                 return -ENODEV;
1408                         }
1409                 }
1410         }
1411         return 0;
1412 }
1413
1414 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1415 {
1416         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1417 }
1418
1419 static inline int guestwidth_to_adjustwidth(int gaw)
1420 {
1421         int agaw;
1422         int r = (gaw - 12) % 9;
1423
1424         if (r == 0)
1425                 agaw = gaw;
1426         else
1427                 agaw = gaw + 9 - r;
1428         if (agaw > 64)
1429                 agaw = 64;
1430         return agaw;
1431 }
1432
1433 static int domain_init(struct dmar_domain *domain, int guest_width)
1434 {
1435         struct intel_iommu *iommu;
1436         int adjust_width, agaw;
1437         unsigned long sagaw;
1438
1439         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1440         spin_lock_init(&domain->iommu_lock);
1441
1442         domain_reserve_special_ranges(domain);
1443
1444         /* calculate AGAW */
1445         iommu = domain_get_iommu(domain);
1446         if (guest_width > cap_mgaw(iommu->cap))
1447                 guest_width = cap_mgaw(iommu->cap);
1448         domain->gaw = guest_width;
1449         adjust_width = guestwidth_to_adjustwidth(guest_width);
1450         agaw = width_to_agaw(adjust_width);
1451         sagaw = cap_sagaw(iommu->cap);
1452         if (!test_bit(agaw, &sagaw)) {
1453                 /* hardware doesn't support it, choose a bigger one */
1454                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1455                 agaw = find_next_bit(&sagaw, 5, agaw);
1456                 if (agaw >= 5)
1457                         return -ENODEV;
1458         }
1459         domain->agaw = agaw;
1460         INIT_LIST_HEAD(&domain->devices);
1461
1462         if (ecap_coherent(iommu->ecap))
1463                 domain->iommu_coherency = 1;
1464         else
1465                 domain->iommu_coherency = 0;
1466
1467         if (ecap_sc_support(iommu->ecap))
1468                 domain->iommu_snooping = 1;
1469         else
1470                 domain->iommu_snooping = 0;
1471
1472         domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1473         domain->iommu_count = 1;
1474         domain->nid = iommu->node;
1475
1476         /* always allocate the top pgd */
1477         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1478         if (!domain->pgd)
1479                 return -ENOMEM;
1480         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1481         return 0;
1482 }
1483
1484 static void domain_exit(struct dmar_domain *domain)
1485 {
1486         struct dmar_drhd_unit *drhd;
1487         struct intel_iommu *iommu;
1488
1489         /* Domain 0 is reserved, so dont process it */
1490         if (!domain)
1491                 return;
1492
1493         /* Flush any lazy unmaps that may reference this domain */
1494         if (!intel_iommu_strict)
1495                 flush_unmaps_timeout(0);
1496
1497         domain_remove_dev_info(domain);
1498         /* destroy iovas */
1499         put_iova_domain(&domain->iovad);
1500
1501         /* clear ptes */
1502         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1503
1504         /* free page tables */
1505         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1506
1507         for_each_active_iommu(iommu, drhd)
1508                 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1509                         iommu_detach_domain(domain, iommu);
1510
1511         free_domain_mem(domain);
1512 }
1513
1514 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1515                                  u8 bus, u8 devfn, int translation)
1516 {
1517         struct context_entry *context;
1518         unsigned long flags;
1519         struct intel_iommu *iommu;
1520         struct dma_pte *pgd;
1521         unsigned long num;
1522         unsigned long ndomains;
1523         int id;
1524         int agaw;
1525         struct device_domain_info *info = NULL;
1526
1527         pr_debug("Set context mapping for %02x:%02x.%d\n",
1528                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1529
1530         BUG_ON(!domain->pgd);
1531         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1532                translation != CONTEXT_TT_MULTI_LEVEL);
1533
1534         iommu = device_to_iommu(segment, bus, devfn);
1535         if (!iommu)
1536                 return -ENODEV;
1537
1538         context = device_to_context_entry(iommu, bus, devfn);
1539         if (!context)
1540                 return -ENOMEM;
1541         spin_lock_irqsave(&iommu->lock, flags);
1542         if (context_present(context)) {
1543                 spin_unlock_irqrestore(&iommu->lock, flags);
1544                 return 0;
1545         }
1546
1547         id = domain->id;
1548         pgd = domain->pgd;
1549
1550         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1551             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1552                 int found = 0;
1553
1554                 /* find an available domain id for this device in iommu */
1555                 ndomains = cap_ndoms(iommu->cap);
1556                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1557                         if (iommu->domains[num] == domain) {
1558                                 id = num;
1559                                 found = 1;
1560                                 break;
1561                         }
1562                 }
1563
1564                 if (found == 0) {
1565                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1566                         if (num >= ndomains) {
1567                                 spin_unlock_irqrestore(&iommu->lock, flags);
1568                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1569                                 return -EFAULT;
1570                         }
1571
1572                         set_bit(num, iommu->domain_ids);
1573                         iommu->domains[num] = domain;
1574                         id = num;
1575                 }
1576
1577                 /* Skip top levels of page tables for
1578                  * iommu which has less agaw than default.
1579                  * Unnecessary for PT mode.
1580                  */
1581                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1582                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1583                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1584                                 if (!dma_pte_present(pgd)) {
1585                                         spin_unlock_irqrestore(&iommu->lock, flags);
1586                                         return -ENOMEM;
1587                                 }
1588                         }
1589                 }
1590         }
1591
1592         context_set_domain_id(context, id);
1593
1594         if (translation != CONTEXT_TT_PASS_THROUGH) {
1595                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1596                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1597                                      CONTEXT_TT_MULTI_LEVEL;
1598         }
1599         /*
1600          * In pass through mode, AW must be programmed to indicate the largest
1601          * AGAW value supported by hardware. And ASR is ignored by hardware.
1602          */
1603         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1604                 context_set_address_width(context, iommu->msagaw);
1605         else {
1606                 context_set_address_root(context, virt_to_phys(pgd));
1607                 context_set_address_width(context, iommu->agaw);
1608         }
1609
1610         context_set_translation_type(context, translation);
1611         context_set_fault_enable(context);
1612         context_set_present(context);
1613         domain_flush_cache(domain, context, sizeof(*context));
1614
1615         /*
1616          * It's a non-present to present mapping. If hardware doesn't cache
1617          * non-present entry we only need to flush the write-buffer. If the
1618          * _does_ cache non-present entries, then it does so in the special
1619          * domain #0, which we have to flush:
1620          */
1621         if (cap_caching_mode(iommu->cap)) {
1622                 iommu->flush.flush_context(iommu, 0,
1623                                            (((u16)bus) << 8) | devfn,
1624                                            DMA_CCMD_MASK_NOBIT,
1625                                            DMA_CCMD_DEVICE_INVL);
1626                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1627         } else {
1628                 iommu_flush_write_buffer(iommu);
1629         }
1630         iommu_enable_dev_iotlb(info);
1631         spin_unlock_irqrestore(&iommu->lock, flags);
1632
1633         spin_lock_irqsave(&domain->iommu_lock, flags);
1634         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1635                 domain->iommu_count++;
1636                 if (domain->iommu_count == 1)
1637                         domain->nid = iommu->node;
1638                 domain_update_iommu_cap(domain);
1639         }
1640         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1641         return 0;
1642 }
1643
1644 static int
1645 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1646                         int translation)
1647 {
1648         int ret;
1649         struct pci_dev *tmp, *parent;
1650
1651         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1652                                          pdev->bus->number, pdev->devfn,
1653                                          translation);
1654         if (ret)
1655                 return ret;
1656
1657         /* dependent device mapping */
1658         tmp = pci_find_upstream_pcie_bridge(pdev);
1659         if (!tmp)
1660                 return 0;
1661         /* Secondary interface's bus number and devfn 0 */
1662         parent = pdev->bus->self;
1663         while (parent != tmp) {
1664                 ret = domain_context_mapping_one(domain,
1665                                                  pci_domain_nr(parent->bus),
1666                                                  parent->bus->number,
1667                                                  parent->devfn, translation);
1668                 if (ret)
1669                         return ret;
1670                 parent = parent->bus->self;
1671         }
1672         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1673                 return domain_context_mapping_one(domain,
1674                                         pci_domain_nr(tmp->subordinate),
1675                                         tmp->subordinate->number, 0,
1676                                         translation);
1677         else /* this is a legacy PCI bridge */
1678                 return domain_context_mapping_one(domain,
1679                                                   pci_domain_nr(tmp->bus),
1680                                                   tmp->bus->number,
1681                                                   tmp->devfn,
1682                                                   translation);
1683 }
1684
1685 static int domain_context_mapped(struct pci_dev *pdev)
1686 {
1687         int ret;
1688         struct pci_dev *tmp, *parent;
1689         struct intel_iommu *iommu;
1690
1691         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1692                                 pdev->devfn);
1693         if (!iommu)
1694                 return -ENODEV;
1695
1696         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1697         if (!ret)
1698                 return ret;
1699         /* dependent device mapping */
1700         tmp = pci_find_upstream_pcie_bridge(pdev);
1701         if (!tmp)
1702                 return ret;
1703         /* Secondary interface's bus number and devfn 0 */
1704         parent = pdev->bus->self;
1705         while (parent != tmp) {
1706                 ret = device_context_mapped(iommu, parent->bus->number,
1707                                             parent->devfn);
1708                 if (!ret)
1709                         return ret;
1710                 parent = parent->bus->self;
1711         }
1712         if (pci_is_pcie(tmp))
1713                 return device_context_mapped(iommu, tmp->subordinate->number,
1714                                              0);
1715         else
1716                 return device_context_mapped(iommu, tmp->bus->number,
1717                                              tmp->devfn);
1718 }
1719
1720 /* Returns a number of VTD pages, but aligned to MM page size */
1721 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1722                                             size_t size)
1723 {
1724         host_addr &= ~PAGE_MASK;
1725         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1726 }
1727
1728 /* Return largest possible superpage level for a given mapping */
1729 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1730                                           unsigned long iov_pfn,
1731                                           unsigned long phy_pfn,
1732                                           unsigned long pages)
1733 {
1734         int support, level = 1;
1735         unsigned long pfnmerge;
1736
1737         support = domain->iommu_superpage;
1738
1739         /* To use a large page, the virtual *and* physical addresses
1740            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1741            of them will mean we have to use smaller pages. So just
1742            merge them and check both at once. */
1743         pfnmerge = iov_pfn | phy_pfn;
1744
1745         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1746                 pages >>= VTD_STRIDE_SHIFT;
1747                 if (!pages)
1748                         break;
1749                 pfnmerge >>= VTD_STRIDE_SHIFT;
1750                 level++;
1751                 support--;
1752         }
1753         return level;
1754 }
1755
1756 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1757                             struct scatterlist *sg, unsigned long phys_pfn,
1758                             unsigned long nr_pages, int prot)
1759 {
1760         struct dma_pte *first_pte = NULL, *pte = NULL;
1761         phys_addr_t uninitialized_var(pteval);
1762         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1763         unsigned long sg_res;
1764         unsigned int largepage_lvl = 0;
1765         unsigned long lvl_pages = 0;
1766
1767         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1768
1769         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1770                 return -EINVAL;
1771
1772         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1773
1774         if (sg)
1775                 sg_res = 0;
1776         else {
1777                 sg_res = nr_pages + 1;
1778                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1779         }
1780
1781         while (nr_pages > 0) {
1782                 uint64_t tmp;
1783
1784                 if (!sg_res) {
1785                         sg_res = aligned_nrpages(sg->offset, sg->length);
1786                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1787                         sg->dma_length = sg->length;
1788                         pteval = page_to_phys(sg_page(sg)) | prot;
1789                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1790                 }
1791
1792                 if (!pte) {
1793                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1794
1795                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1796                         if (!pte)
1797                                 return -ENOMEM;
1798                         /* It is large page*/
1799                         if (largepage_lvl > 1)
1800                                 pteval |= DMA_PTE_LARGE_PAGE;
1801                         else
1802                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1803
1804                 }
1805                 /* We don't need lock here, nobody else
1806                  * touches the iova range
1807                  */
1808                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1809                 if (tmp) {
1810                         static int dumps = 5;
1811                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1812                                iov_pfn, tmp, (unsigned long long)pteval);
1813                         if (dumps) {
1814                                 dumps--;
1815                                 debug_dma_dump_mappings(NULL);
1816                         }
1817                         WARN_ON(1);
1818                 }
1819
1820                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1821
1822                 BUG_ON(nr_pages < lvl_pages);
1823                 BUG_ON(sg_res < lvl_pages);
1824
1825                 nr_pages -= lvl_pages;
1826                 iov_pfn += lvl_pages;
1827                 phys_pfn += lvl_pages;
1828                 pteval += lvl_pages * VTD_PAGE_SIZE;
1829                 sg_res -= lvl_pages;
1830
1831                 /* If the next PTE would be the first in a new page, then we
1832                    need to flush the cache on the entries we've just written.
1833                    And then we'll need to recalculate 'pte', so clear it and
1834                    let it get set again in the if (!pte) block above.
1835
1836                    If we're done (!nr_pages) we need to flush the cache too.
1837
1838                    Also if we've been setting superpages, we may need to
1839                    recalculate 'pte' and switch back to smaller pages for the
1840                    end of the mapping, if the trailing size is not enough to
1841                    use another superpage (i.e. sg_res < lvl_pages). */
1842                 pte++;
1843                 if (!nr_pages || first_pte_in_page(pte) ||
1844                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
1845                         domain_flush_cache(domain, first_pte,
1846                                            (void *)pte - (void *)first_pte);
1847                         pte = NULL;
1848                 }
1849
1850                 if (!sg_res && nr_pages)
1851                         sg = sg_next(sg);
1852         }
1853         return 0;
1854 }
1855
1856 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1857                                     struct scatterlist *sg, unsigned long nr_pages,
1858                                     int prot)
1859 {
1860         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1861 }
1862
1863 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1864                                      unsigned long phys_pfn, unsigned long nr_pages,
1865                                      int prot)
1866 {
1867         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1868 }
1869
1870 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1871 {
1872         if (!iommu)
1873                 return;
1874
1875         clear_context_table(iommu, bus, devfn);
1876         iommu->flush.flush_context(iommu, 0, 0, 0,
1877                                            DMA_CCMD_GLOBAL_INVL);
1878         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1879 }
1880
1881 static void domain_remove_dev_info(struct dmar_domain *domain)
1882 {
1883         struct device_domain_info *info;
1884         unsigned long flags;
1885         struct intel_iommu *iommu;
1886
1887         spin_lock_irqsave(&device_domain_lock, flags);
1888         while (!list_empty(&domain->devices)) {
1889                 info = list_entry(domain->devices.next,
1890                         struct device_domain_info, link);
1891                 list_del(&info->link);
1892                 list_del(&info->global);
1893                 if (info->dev)
1894                         info->dev->dev.archdata.iommu = NULL;
1895                 spin_unlock_irqrestore(&device_domain_lock, flags);
1896
1897                 iommu_disable_dev_iotlb(info);
1898                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1899                 iommu_detach_dev(iommu, info->bus, info->devfn);
1900                 free_devinfo_mem(info);
1901
1902                 spin_lock_irqsave(&device_domain_lock, flags);
1903         }
1904         spin_unlock_irqrestore(&device_domain_lock, flags);
1905 }
1906
1907 /*
1908  * find_domain
1909  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1910  */
1911 static struct dmar_domain *
1912 find_domain(struct pci_dev *pdev)
1913 {
1914         struct device_domain_info *info;
1915
1916         /* No lock here, assumes no domain exit in normal case */
1917         info = pdev->dev.archdata.iommu;
1918         if (info)
1919                 return info->domain;
1920         return NULL;
1921 }
1922
1923 /* domain is initialized */
1924 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1925 {
1926         struct dmar_domain *domain, *found = NULL;
1927         struct intel_iommu *iommu;
1928         struct dmar_drhd_unit *drhd;
1929         struct device_domain_info *info, *tmp;
1930         struct pci_dev *dev_tmp;
1931         unsigned long flags;
1932         int bus = 0, devfn = 0;
1933         int segment;
1934         int ret;
1935
1936         domain = find_domain(pdev);
1937         if (domain)
1938                 return domain;
1939
1940         segment = pci_domain_nr(pdev->bus);
1941
1942         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1943         if (dev_tmp) {
1944                 if (pci_is_pcie(dev_tmp)) {
1945                         bus = dev_tmp->subordinate->number;
1946                         devfn = 0;
1947                 } else {
1948                         bus = dev_tmp->bus->number;
1949                         devfn = dev_tmp->devfn;
1950                 }
1951                 spin_lock_irqsave(&device_domain_lock, flags);
1952                 list_for_each_entry(info, &device_domain_list, global) {
1953                         if (info->segment == segment &&
1954                             info->bus == bus && info->devfn == devfn) {
1955                                 found = info->domain;
1956                                 break;
1957                         }
1958                 }
1959                 spin_unlock_irqrestore(&device_domain_lock, flags);
1960                 /* pcie-pci bridge already has a domain, uses it */
1961                 if (found) {
1962                         domain = found;
1963                         goto found_domain;
1964                 }
1965         }
1966
1967         domain = alloc_domain();
1968         if (!domain)
1969                 goto error;
1970
1971         /* Allocate new domain for the device */
1972         drhd = dmar_find_matched_drhd_unit(pdev);
1973         if (!drhd) {
1974                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1975                         pci_name(pdev));
1976                 return NULL;
1977         }
1978         iommu = drhd->iommu;
1979
1980         ret = iommu_attach_domain(domain, iommu);
1981         if (ret) {
1982                 free_domain_mem(domain);
1983                 goto error;
1984         }
1985
1986         if (domain_init(domain, gaw)) {
1987                 domain_exit(domain);
1988                 goto error;
1989         }
1990
1991         /* register pcie-to-pci device */
1992         if (dev_tmp) {
1993                 info = alloc_devinfo_mem();
1994                 if (!info) {
1995                         domain_exit(domain);
1996                         goto error;
1997                 }
1998                 info->segment = segment;
1999                 info->bus = bus;
2000                 info->devfn = devfn;
2001                 info->dev = NULL;
2002                 info->domain = domain;
2003                 /* This domain is shared by devices under p2p bridge */
2004                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2005
2006                 /* pcie-to-pci bridge already has a domain, uses it */
2007                 found = NULL;
2008                 spin_lock_irqsave(&device_domain_lock, flags);
2009                 list_for_each_entry(tmp, &device_domain_list, global) {
2010                         if (tmp->segment == segment &&
2011                             tmp->bus == bus && tmp->devfn == devfn) {
2012                                 found = tmp->domain;
2013                                 break;
2014                         }
2015                 }
2016                 if (found) {
2017                         spin_unlock_irqrestore(&device_domain_lock, flags);
2018                         free_devinfo_mem(info);
2019                         domain_exit(domain);
2020                         domain = found;
2021                 } else {
2022                         list_add(&info->link, &domain->devices);
2023                         list_add(&info->global, &device_domain_list);
2024                         spin_unlock_irqrestore(&device_domain_lock, flags);
2025                 }
2026         }
2027
2028 found_domain:
2029         info = alloc_devinfo_mem();
2030         if (!info)
2031                 goto error;
2032         info->segment = segment;
2033         info->bus = pdev->bus->number;
2034         info->devfn = pdev->devfn;
2035         info->dev = pdev;
2036         info->domain = domain;
2037         spin_lock_irqsave(&device_domain_lock, flags);
2038         /* somebody is fast */
2039         found = find_domain(pdev);
2040         if (found != NULL) {
2041                 spin_unlock_irqrestore(&device_domain_lock, flags);
2042                 if (found != domain) {
2043                         domain_exit(domain);
2044                         domain = found;
2045                 }
2046                 free_devinfo_mem(info);
2047                 return domain;
2048         }
2049         list_add(&info->link, &domain->devices);
2050         list_add(&info->global, &device_domain_list);
2051         pdev->dev.archdata.iommu = info;
2052         spin_unlock_irqrestore(&device_domain_lock, flags);
2053         return domain;
2054 error:
2055         /* recheck it here, maybe others set it */
2056         return find_domain(pdev);
2057 }
2058
2059 static int iommu_identity_mapping;
2060 #define IDENTMAP_ALL            1
2061 #define IDENTMAP_GFX            2
2062 #define IDENTMAP_AZALIA         4
2063
2064 static int iommu_domain_identity_map(struct dmar_domain *domain,
2065                                      unsigned long long start,
2066                                      unsigned long long end)
2067 {
2068         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2069         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2070
2071         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2072                           dma_to_mm_pfn(last_vpfn))) {
2073                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2074                 return -ENOMEM;
2075         }
2076
2077         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2078                  start, end, domain->id);
2079         /*
2080          * RMRR range might have overlap with physical memory range,
2081          * clear it first
2082          */
2083         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2084
2085         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2086                                   last_vpfn - first_vpfn + 1,
2087                                   DMA_PTE_READ|DMA_PTE_WRITE);
2088 }
2089
2090 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2091                                       unsigned long long start,
2092                                       unsigned long long end)
2093 {
2094         struct dmar_domain *domain;
2095         int ret;
2096
2097         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2098         if (!domain)
2099                 return -ENOMEM;
2100
2101         /* For _hardware_ passthrough, don't bother. But for software
2102            passthrough, we do it anyway -- it may indicate a memory
2103            range which is reserved in E820, so which didn't get set
2104            up to start with in si_domain */
2105         if (domain == si_domain && hw_pass_through) {
2106                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2107                        pci_name(pdev), start, end);
2108                 return 0;
2109         }
2110
2111         printk(KERN_INFO
2112                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2113                pci_name(pdev), start, end);
2114
2115         if (end < start) {
2116                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2117                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2118                         dmi_get_system_info(DMI_BIOS_VENDOR),
2119                         dmi_get_system_info(DMI_BIOS_VERSION),
2120                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2121                 ret = -EIO;
2122                 goto error;
2123         }
2124
2125         if (end >> agaw_to_width(domain->agaw)) {
2126                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2127                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2128                      agaw_to_width(domain->agaw),
2129                      dmi_get_system_info(DMI_BIOS_VENDOR),
2130                      dmi_get_system_info(DMI_BIOS_VERSION),
2131                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2132                 ret = -EIO;
2133                 goto error;
2134         }
2135
2136         ret = iommu_domain_identity_map(domain, start, end);
2137         if (ret)
2138                 goto error;
2139
2140         /* context entry init */
2141         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2142         if (ret)
2143                 goto error;
2144
2145         return 0;
2146
2147  error:
2148         domain_exit(domain);
2149         return ret;
2150 }
2151
2152 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2153         struct pci_dev *pdev)
2154 {
2155         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2156                 return 0;
2157         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2158                 rmrr->end_address);
2159 }
2160
2161 #ifdef CONFIG_DMAR_FLOPPY_WA
2162 static inline void iommu_prepare_isa(void)
2163 {
2164         struct pci_dev *pdev;
2165         int ret;
2166
2167         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2168         if (!pdev)
2169                 return;
2170
2171         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2172         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2173
2174         if (ret)
2175                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2176                        "floppy might not work\n");
2177
2178 }
2179 #else
2180 static inline void iommu_prepare_isa(void)
2181 {
2182         return;
2183 }
2184 #endif /* !CONFIG_DMAR_FLPY_WA */
2185
2186 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2187
2188 static int __init si_domain_work_fn(unsigned long start_pfn,
2189                                     unsigned long end_pfn, void *datax)
2190 {
2191         int *ret = datax;
2192
2193         *ret = iommu_domain_identity_map(si_domain,
2194                                          (uint64_t)start_pfn << PAGE_SHIFT,
2195                                          (uint64_t)end_pfn << PAGE_SHIFT);
2196         return *ret;
2197
2198 }
2199
2200 static int __init si_domain_init(int hw)
2201 {
2202         struct dmar_drhd_unit *drhd;
2203         struct intel_iommu *iommu;
2204         int nid, ret = 0;
2205
2206         si_domain = alloc_domain();
2207         if (!si_domain)
2208                 return -EFAULT;
2209
2210         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2211
2212         for_each_active_iommu(iommu, drhd) {
2213                 ret = iommu_attach_domain(si_domain, iommu);
2214                 if (ret) {
2215                         domain_exit(si_domain);
2216                         return -EFAULT;
2217                 }
2218         }
2219
2220         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2221                 domain_exit(si_domain);
2222                 return -EFAULT;
2223         }
2224
2225         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2226
2227         if (hw)
2228                 return 0;
2229
2230         for_each_online_node(nid) {
2231                 work_with_active_regions(nid, si_domain_work_fn, &ret);
2232                 if (ret)
2233                         return ret;
2234         }
2235
2236         return 0;
2237 }
2238
2239 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2240                                           struct pci_dev *pdev);
2241 static int identity_mapping(struct pci_dev *pdev)
2242 {
2243         struct device_domain_info *info;
2244
2245         if (likely(!iommu_identity_mapping))
2246                 return 0;
2247
2248         info = pdev->dev.archdata.iommu;
2249         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2250                 return (info->domain == si_domain);
2251
2252         return 0;
2253 }
2254
2255 static int domain_add_dev_info(struct dmar_domain *domain,
2256                                struct pci_dev *pdev,
2257                                int translation)
2258 {
2259         struct device_domain_info *info;
2260         unsigned long flags;
2261         int ret;
2262
2263         info = alloc_devinfo_mem();
2264         if (!info)
2265                 return -ENOMEM;
2266
2267         ret = domain_context_mapping(domain, pdev, translation);
2268         if (ret) {
2269                 free_devinfo_mem(info);
2270                 return ret;
2271         }
2272
2273         info->segment = pci_domain_nr(pdev->bus);
2274         info->bus = pdev->bus->number;
2275         info->devfn = pdev->devfn;
2276         info->dev = pdev;
2277         info->domain = domain;
2278
2279         spin_lock_irqsave(&device_domain_lock, flags);
2280         list_add(&info->link, &domain->devices);
2281         list_add(&info->global, &device_domain_list);
2282         pdev->dev.archdata.iommu = info;
2283         spin_unlock_irqrestore(&device_domain_lock, flags);
2284
2285         return 0;
2286 }
2287
2288 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2289 {
2290         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2291                 return 1;
2292
2293         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2294                 return 1;
2295
2296         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2297                 return 0;
2298
2299         /*
2300          * We want to start off with all devices in the 1:1 domain, and
2301          * take them out later if we find they can't access all of memory.
2302          *
2303          * However, we can't do this for PCI devices behind bridges,
2304          * because all PCI devices behind the same bridge will end up
2305          * with the same source-id on their transactions.
2306          *
2307          * Practically speaking, we can't change things around for these
2308          * devices at run-time, because we can't be sure there'll be no
2309          * DMA transactions in flight for any of their siblings.
2310          *
2311          * So PCI devices (unless they're on the root bus) as well as
2312          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2313          * the 1:1 domain, just in _case_ one of their siblings turns out
2314          * not to be able to map all of memory.
2315          */
2316         if (!pci_is_pcie(pdev)) {
2317                 if (!pci_is_root_bus(pdev->bus))
2318                         return 0;
2319                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2320                         return 0;
2321         } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2322                 return 0;
2323
2324         /*
2325          * At boot time, we don't yet know if devices will be 64-bit capable.
2326          * Assume that they will -- if they turn out not to be, then we can
2327          * take them out of the 1:1 domain later.
2328          */
2329         if (!startup) {
2330                 /*
2331                  * If the device's dma_mask is less than the system's memory
2332                  * size then this is not a candidate for identity mapping.
2333                  */
2334                 u64 dma_mask = pdev->dma_mask;
2335
2336                 if (pdev->dev.coherent_dma_mask &&
2337                     pdev->dev.coherent_dma_mask < dma_mask)
2338                         dma_mask = pdev->dev.coherent_dma_mask;
2339
2340                 return dma_mask >= dma_get_required_mask(&pdev->dev);
2341         }
2342
2343         return 1;
2344 }
2345
2346 static int __init iommu_prepare_static_identity_mapping(int hw)
2347 {
2348         struct pci_dev *pdev = NULL;
2349         int ret;
2350
2351         ret = si_domain_init(hw);
2352         if (ret)
2353                 return -EFAULT;
2354
2355         for_each_pci_dev(pdev) {
2356                 /* Skip Host/PCI Bridge devices */
2357                 if (IS_BRIDGE_HOST_DEVICE(pdev))
2358                         continue;
2359                 if (iommu_should_identity_map(pdev, 1)) {
2360                         printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2361                                hw ? "hardware" : "software", pci_name(pdev));
2362
2363                         ret = domain_add_dev_info(si_domain, pdev,
2364                                                      hw ? CONTEXT_TT_PASS_THROUGH :
2365                                                      CONTEXT_TT_MULTI_LEVEL);
2366                         if (ret)
2367                                 return ret;
2368                 }
2369         }
2370
2371         return 0;
2372 }
2373
2374 static int __init init_dmars(void)
2375 {
2376         struct dmar_drhd_unit *drhd;
2377         struct dmar_rmrr_unit *rmrr;
2378         struct pci_dev *pdev;
2379         struct intel_iommu *iommu;
2380         int i, ret;
2381
2382         /*
2383          * for each drhd
2384          *    allocate root
2385          *    initialize and program root entry to not present
2386          * endfor
2387          */
2388         for_each_drhd_unit(drhd) {
2389                 g_num_of_iommus++;
2390                 /*
2391                  * lock not needed as this is only incremented in the single
2392                  * threaded kernel __init code path all other access are read
2393                  * only
2394                  */
2395         }
2396
2397         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2398                         GFP_KERNEL);
2399         if (!g_iommus) {
2400                 printk(KERN_ERR "Allocating global iommu array failed\n");
2401                 ret = -ENOMEM;
2402                 goto error;
2403         }
2404
2405         deferred_flush = kzalloc(g_num_of_iommus *
2406                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2407         if (!deferred_flush) {
2408                 ret = -ENOMEM;
2409                 goto error;
2410         }
2411
2412         for_each_drhd_unit(drhd) {
2413                 if (drhd->ignored)
2414                         continue;
2415
2416                 iommu = drhd->iommu;
2417                 g_iommus[iommu->seq_id] = iommu;
2418
2419                 ret = iommu_init_domains(iommu);
2420                 if (ret)
2421                         goto error;
2422
2423                 /*
2424                  * TBD:
2425                  * we could share the same root & context tables
2426                  * among all IOMMU's. Need to Split it later.
2427                  */
2428                 ret = iommu_alloc_root_entry(iommu);
2429                 if (ret) {
2430                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2431                         goto error;
2432                 }
2433                 if (!ecap_pass_through(iommu->ecap))
2434                         hw_pass_through = 0;
2435         }
2436
2437         /*
2438          * Start from the sane iommu hardware state.
2439          */
2440         for_each_drhd_unit(drhd) {
2441                 if (drhd->ignored)
2442                         continue;
2443
2444                 iommu = drhd->iommu;
2445
2446                 /*
2447                  * If the queued invalidation is already initialized by us
2448                  * (for example, while enabling interrupt-remapping) then
2449                  * we got the things already rolling from a sane state.
2450                  */
2451                 if (iommu->qi)
2452                         continue;
2453
2454                 /*
2455                  * Clear any previous faults.
2456                  */
2457                 dmar_fault(-1, iommu);
2458                 /*
2459                  * Disable queued invalidation if supported and already enabled
2460                  * before OS handover.
2461                  */
2462                 dmar_disable_qi(iommu);
2463         }
2464
2465         for_each_drhd_unit(drhd) {
2466                 if (drhd->ignored)
2467                         continue;
2468
2469                 iommu = drhd->iommu;
2470
2471                 if (dmar_enable_qi(iommu)) {
2472                         /*
2473                          * Queued Invalidate not enabled, use Register Based
2474                          * Invalidate
2475                          */
2476                         iommu->flush.flush_context = __iommu_flush_context;
2477                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2478                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2479                                "invalidation\n",
2480                                 iommu->seq_id,
2481                                (unsigned long long)drhd->reg_base_addr);
2482                 } else {
2483                         iommu->flush.flush_context = qi_flush_context;
2484                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2485                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2486                                "invalidation\n",
2487                                 iommu->seq_id,
2488                                (unsigned long long)drhd->reg_base_addr);
2489                 }
2490         }
2491
2492         if (iommu_pass_through)
2493                 iommu_identity_mapping |= IDENTMAP_ALL;
2494
2495 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2496         iommu_identity_mapping |= IDENTMAP_GFX;
2497 #endif
2498
2499         check_tylersburg_isoch();
2500
2501         /*
2502          * If pass through is not set or not enabled, setup context entries for
2503          * identity mappings for rmrr, gfx, and isa and may fall back to static
2504          * identity mapping if iommu_identity_mapping is set.
2505          */
2506         if (iommu_identity_mapping) {
2507                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2508                 if (ret) {
2509                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2510                         goto error;
2511                 }
2512         }
2513         /*
2514          * For each rmrr
2515          *   for each dev attached to rmrr
2516          *   do
2517          *     locate drhd for dev, alloc domain for dev
2518          *     allocate free domain
2519          *     allocate page table entries for rmrr
2520          *     if context not allocated for bus
2521          *           allocate and init context
2522          *           set present in root table for this bus
2523          *     init context with domain, translation etc
2524          *    endfor
2525          * endfor
2526          */
2527         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2528         for_each_rmrr_units(rmrr) {
2529                 for (i = 0; i < rmrr->devices_cnt; i++) {
2530                         pdev = rmrr->devices[i];
2531                         /*
2532                          * some BIOS lists non-exist devices in DMAR
2533                          * table.
2534                          */
2535                         if (!pdev)
2536                                 continue;
2537                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2538                         if (ret)
2539                                 printk(KERN_ERR
2540                                        "IOMMU: mapping reserved region failed\n");
2541                 }
2542         }
2543
2544         iommu_prepare_isa();
2545
2546         /*
2547          * for each drhd
2548          *   enable fault log
2549          *   global invalidate context cache
2550          *   global invalidate iotlb
2551          *   enable translation
2552          */
2553         for_each_drhd_unit(drhd) {
2554                 if (drhd->ignored) {
2555                         /*
2556                          * we always have to disable PMRs or DMA may fail on
2557                          * this device
2558                          */
2559                         if (force_on)
2560                                 iommu_disable_protect_mem_regions(drhd->iommu);
2561                         continue;
2562                 }
2563                 iommu = drhd->iommu;
2564
2565                 iommu_flush_write_buffer(iommu);
2566
2567                 ret = dmar_set_interrupt(iommu);
2568                 if (ret)
2569                         goto error;
2570
2571                 iommu_set_root_entry(iommu);
2572
2573                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2574                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2575
2576                 ret = iommu_enable_translation(iommu);
2577                 if (ret)
2578                         goto error;
2579
2580                 iommu_disable_protect_mem_regions(iommu);
2581         }
2582
2583         return 0;
2584 error:
2585         for_each_drhd_unit(drhd) {
2586                 if (drhd->ignored)
2587                         continue;
2588                 iommu = drhd->iommu;
2589                 free_iommu(iommu);
2590         }
2591         kfree(g_iommus);
2592         return ret;
2593 }
2594
2595 /* This takes a number of _MM_ pages, not VTD pages */
2596 static struct iova *intel_alloc_iova(struct device *dev,
2597                                      struct dmar_domain *domain,
2598                                      unsigned long nrpages, uint64_t dma_mask)
2599 {
2600         struct pci_dev *pdev = to_pci_dev(dev);
2601         struct iova *iova = NULL;
2602
2603         /* Restrict dma_mask to the width that the iommu can handle */
2604         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2605
2606         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2607                 /*
2608                  * First try to allocate an io virtual address in
2609                  * DMA_BIT_MASK(32) and if that fails then try allocating
2610                  * from higher range
2611                  */
2612                 iova = alloc_iova(&domain->iovad, nrpages,
2613                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2614                 if (iova)
2615                         return iova;
2616         }
2617         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2618         if (unlikely(!iova)) {
2619                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2620                        nrpages, pci_name(pdev));
2621                 return NULL;
2622         }
2623
2624         return iova;
2625 }
2626
2627 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2628 {
2629         struct dmar_domain *domain;
2630         int ret;
2631
2632         domain = get_domain_for_dev(pdev,
2633                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2634         if (!domain) {
2635                 printk(KERN_ERR
2636                         "Allocating domain for %s failed", pci_name(pdev));
2637                 return NULL;
2638         }
2639
2640         /* make sure context mapping is ok */
2641         if (unlikely(!domain_context_mapped(pdev))) {
2642                 ret = domain_context_mapping(domain, pdev,
2643                                              CONTEXT_TT_MULTI_LEVEL);
2644                 if (ret) {
2645                         printk(KERN_ERR
2646                                 "Domain context map for %s failed",
2647                                 pci_name(pdev));
2648                         return NULL;
2649                 }
2650         }
2651
2652         return domain;
2653 }
2654
2655 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2656 {
2657         struct device_domain_info *info;
2658
2659         /* No lock here, assumes no domain exit in normal case */
2660         info = dev->dev.archdata.iommu;
2661         if (likely(info))
2662                 return info->domain;
2663
2664         return __get_valid_domain_for_dev(dev);
2665 }
2666
2667 static int iommu_dummy(struct pci_dev *pdev)
2668 {
2669         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2670 }
2671
2672 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2673 static int iommu_no_mapping(struct device *dev)
2674 {
2675         struct pci_dev *pdev;
2676         int found;
2677
2678         if (unlikely(dev->bus != &pci_bus_type))
2679                 return 1;
2680
2681         pdev = to_pci_dev(dev);
2682         if (iommu_dummy(pdev))
2683                 return 1;
2684
2685         if (!iommu_identity_mapping)
2686                 return 0;
2687
2688         found = identity_mapping(pdev);
2689         if (found) {
2690                 if (iommu_should_identity_map(pdev, 0))
2691                         return 1;
2692                 else {
2693                         /*
2694                          * 32 bit DMA is removed from si_domain and fall back
2695                          * to non-identity mapping.
2696                          */
2697                         domain_remove_one_dev_info(si_domain, pdev);
2698                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2699                                pci_name(pdev));
2700                         return 0;
2701                 }
2702         } else {
2703                 /*
2704                  * In case of a detached 64 bit DMA device from vm, the device
2705                  * is put into si_domain for identity mapping.
2706                  */
2707                 if (iommu_should_identity_map(pdev, 0)) {
2708                         int ret;
2709                         ret = domain_add_dev_info(si_domain, pdev,
2710                                                   hw_pass_through ?
2711                                                   CONTEXT_TT_PASS_THROUGH :
2712                                                   CONTEXT_TT_MULTI_LEVEL);
2713                         if (!ret) {
2714                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2715                                        pci_name(pdev));
2716                                 return 1;
2717                         }
2718                 }
2719         }
2720
2721         return 0;
2722 }
2723
2724 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2725                                      size_t size, int dir, u64 dma_mask)
2726 {
2727         struct pci_dev *pdev = to_pci_dev(hwdev);
2728         struct dmar_domain *domain;
2729         phys_addr_t start_paddr;
2730         struct iova *iova;
2731         int prot = 0;
2732         int ret;
2733         struct intel_iommu *iommu;
2734         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2735
2736         BUG_ON(dir == DMA_NONE);
2737
2738         if (iommu_no_mapping(hwdev))
2739                 return paddr;
2740
2741         domain = get_valid_domain_for_dev(pdev);
2742         if (!domain)
2743                 return 0;
2744
2745         iommu = domain_get_iommu(domain);
2746         size = aligned_nrpages(paddr, size);
2747
2748         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2749         if (!iova)
2750                 goto error;
2751
2752         /*
2753          * Check if DMAR supports zero-length reads on write only
2754          * mappings..
2755          */
2756         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2757                         !cap_zlr(iommu->cap))
2758                 prot |= DMA_PTE_READ;
2759         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2760                 prot |= DMA_PTE_WRITE;
2761         /*
2762          * paddr - (paddr + size) might be partial page, we should map the whole
2763          * page.  Note: if two part of one page are separately mapped, we
2764          * might have two guest_addr mapping to the same host paddr, but this
2765          * is not a big problem
2766          */
2767         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2768                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2769         if (ret)
2770                 goto error;
2771
2772         /* it's a non-present to present mapping. Only flush if caching mode */
2773         if (cap_caching_mode(iommu->cap))
2774                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2775         else
2776                 iommu_flush_write_buffer(iommu);
2777
2778         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2779         start_paddr += paddr & ~PAGE_MASK;
2780         return start_paddr;
2781
2782 error:
2783         if (iova)
2784                 __free_iova(&domain->iovad, iova);
2785         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2786                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2787         return 0;
2788 }
2789
2790 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2791                                  unsigned long offset, size_t size,
2792                                  enum dma_data_direction dir,
2793                                  struct dma_attrs *attrs)
2794 {
2795         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2796                                   dir, to_pci_dev(dev)->dma_mask);
2797 }
2798
2799 static void flush_unmaps(void)
2800 {
2801         int i, j;
2802
2803         timer_on = 0;
2804
2805         /* just flush them all */
2806         for (i = 0; i < g_num_of_iommus; i++) {
2807                 struct intel_iommu *iommu = g_iommus[i];
2808                 if (!iommu)
2809                         continue;
2810
2811                 if (!deferred_flush[i].next)
2812                         continue;
2813
2814                 /* In caching mode, global flushes turn emulation expensive */
2815                 if (!cap_caching_mode(iommu->cap))
2816                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2817                                          DMA_TLB_GLOBAL_FLUSH);
2818                 for (j = 0; j < deferred_flush[i].next; j++) {
2819                         unsigned long mask;
2820                         struct iova *iova = deferred_flush[i].iova[j];
2821                         struct dmar_domain *domain = deferred_flush[i].domain[j];
2822
2823                         /* On real hardware multiple invalidations are expensive */
2824                         if (cap_caching_mode(iommu->cap))
2825                                 iommu_flush_iotlb_psi(iommu, domain->id,
2826                                 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2827                         else {
2828                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2829                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2830                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2831                         }
2832                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2833                 }
2834                 deferred_flush[i].next = 0;
2835         }
2836
2837         list_size = 0;
2838 }
2839
2840 static void flush_unmaps_timeout(unsigned long data)
2841 {
2842         unsigned long flags;
2843
2844         spin_lock_irqsave(&async_umap_flush_lock, flags);
2845         flush_unmaps();
2846         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2847 }
2848
2849 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2850 {
2851         unsigned long flags;
2852         int next, iommu_id;
2853         struct intel_iommu *iommu;
2854
2855         spin_lock_irqsave(&async_umap_flush_lock, flags);
2856         if (list_size == HIGH_WATER_MARK)
2857                 flush_unmaps();
2858
2859         iommu = domain_get_iommu(dom);
2860         iommu_id = iommu->seq_id;
2861
2862         next = deferred_flush[iommu_id].next;
2863         deferred_flush[iommu_id].domain[next] = dom;
2864         deferred_flush[iommu_id].iova[next] = iova;
2865         deferred_flush[iommu_id].next++;
2866
2867         if (!timer_on) {
2868                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2869                 timer_on = 1;
2870         }
2871         list_size++;
2872         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2873 }
2874
2875 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2876                              size_t size, enum dma_data_direction dir,
2877                              struct dma_attrs *attrs)
2878 {
2879         struct pci_dev *pdev = to_pci_dev(dev);
2880         struct dmar_domain *domain;
2881         unsigned long start_pfn, last_pfn;
2882         struct iova *iova;
2883         struct intel_iommu *iommu;
2884
2885         if (iommu_no_mapping(dev))
2886                 return;
2887
2888         domain = find_domain(pdev);
2889         BUG_ON(!domain);
2890
2891         iommu = domain_get_iommu(domain);
2892
2893         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2894         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2895                       (unsigned long long)dev_addr))
2896                 return;
2897
2898         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2899         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2900
2901         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2902                  pci_name(pdev), start_pfn, last_pfn);
2903
2904         /*  clear the whole page */
2905         dma_pte_clear_range(domain, start_pfn, last_pfn);
2906
2907         /* free page tables */
2908         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2909
2910         if (intel_iommu_strict) {
2911                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2912                                       last_pfn - start_pfn + 1, 0);
2913                 /* free iova */
2914                 __free_iova(&domain->iovad, iova);
2915         } else {
2916                 add_unmap(domain, iova);
2917                 /*
2918                  * queue up the release of the unmap to save the 1/6th of the
2919                  * cpu used up by the iotlb flush operation...
2920                  */
2921         }
2922 }
2923
2924 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2925                                   dma_addr_t *dma_handle, gfp_t flags)
2926 {
2927         void *vaddr;
2928         int order;
2929
2930         size = PAGE_ALIGN(size);
2931         order = get_order(size);
2932
2933         if (!iommu_no_mapping(hwdev))
2934                 flags &= ~(GFP_DMA | GFP_DMA32);
2935         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2936                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2937                         flags |= GFP_DMA;
2938                 else
2939                         flags |= GFP_DMA32;
2940         }
2941
2942         vaddr = (void *)__get_free_pages(flags, order);
2943         if (!vaddr)
2944                 return NULL;
2945         memset(vaddr, 0, size);
2946
2947         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2948                                          DMA_BIDIRECTIONAL,
2949                                          hwdev->coherent_dma_mask);
2950         if (*dma_handle)
2951                 return vaddr;
2952         free_pages((unsigned long)vaddr, order);
2953         return NULL;
2954 }
2955
2956 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2957                                 dma_addr_t dma_handle)
2958 {
2959         int order;
2960
2961         size = PAGE_ALIGN(size);
2962         order = get_order(size);
2963
2964         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2965         free_pages((unsigned long)vaddr, order);
2966 }
2967
2968 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2969                            int nelems, enum dma_data_direction dir,
2970                            struct dma_attrs *attrs)
2971 {
2972         struct pci_dev *pdev = to_pci_dev(hwdev);
2973         struct dmar_domain *domain;
2974         unsigned long start_pfn, last_pfn;
2975         struct iova *iova;
2976         struct intel_iommu *iommu;
2977
2978         if (iommu_no_mapping(hwdev))
2979                 return;
2980
2981         domain = find_domain(pdev);
2982         BUG_ON(!domain);
2983
2984         iommu = domain_get_iommu(domain);
2985
2986         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2987         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2988                       (unsigned long long)sglist[0].dma_address))
2989                 return;
2990
2991         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2992         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2993
2994         /*  clear the whole page */
2995         dma_pte_clear_range(domain, start_pfn, last_pfn);
2996
2997         /* free page tables */
2998         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2999
3000         if (intel_iommu_strict) {
3001                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3002                                       last_pfn - start_pfn + 1, 0);
3003                 /* free iova */
3004                 __free_iova(&domain->iovad, iova);
3005         } else {
3006                 add_unmap(domain, iova);
3007                 /*
3008                  * queue up the release of the unmap to save the 1/6th of the
3009                  * cpu used up by the iotlb flush operation...
3010                  */
3011         }
3012 }
3013
3014 static int intel_nontranslate_map_sg(struct device *hddev,
3015         struct scatterlist *sglist, int nelems, int dir)
3016 {
3017         int i;
3018         struct scatterlist *sg;
3019
3020         for_each_sg(sglist, sg, nelems, i) {
3021                 BUG_ON(!sg_page(sg));
3022                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3023                 sg->dma_length = sg->length;
3024         }
3025         return nelems;
3026 }
3027
3028 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3029                         enum dma_data_direction dir, struct dma_attrs *attrs)
3030 {
3031         int i;
3032         struct pci_dev *pdev = to_pci_dev(hwdev);
3033         struct dmar_domain *domain;
3034         size_t size = 0;
3035         int prot = 0;
3036         struct iova *iova = NULL;
3037         int ret;
3038         struct scatterlist *sg;
3039         unsigned long start_vpfn;
3040         struct intel_iommu *iommu;
3041
3042         BUG_ON(dir == DMA_NONE);
3043         if (iommu_no_mapping(hwdev))
3044                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3045
3046         domain = get_valid_domain_for_dev(pdev);
3047         if (!domain)
3048                 return 0;
3049
3050         iommu = domain_get_iommu(domain);
3051
3052         for_each_sg(sglist, sg, nelems, i)
3053                 size += aligned_nrpages(sg->offset, sg->length);
3054
3055         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3056                                 pdev->dma_mask);
3057         if (!iova) {
3058                 sglist->dma_length = 0;
3059                 return 0;
3060         }
3061
3062         /*
3063          * Check if DMAR supports zero-length reads on write only
3064          * mappings..
3065          */
3066         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3067                         !cap_zlr(iommu->cap))
3068                 prot |= DMA_PTE_READ;
3069         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3070                 prot |= DMA_PTE_WRITE;
3071
3072         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3073
3074         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3075         if (unlikely(ret)) {
3076                 /*  clear the page */
3077                 dma_pte_clear_range(domain, start_vpfn,
3078                                     start_vpfn + size - 1);
3079                 /* free page tables */
3080                 dma_pte_free_pagetable(domain, start_vpfn,
3081                                        start_vpfn + size - 1);
3082                 /* free iova */
3083                 __free_iova(&domain->iovad, iova);
3084                 return 0;
3085         }
3086
3087         /* it's a non-present to present mapping. Only flush if caching mode */
3088         if (cap_caching_mode(iommu->cap))
3089                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3090         else
3091                 iommu_flush_write_buffer(iommu);
3092
3093         return nelems;
3094 }
3095
3096 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3097 {
3098         return !dma_addr;
3099 }
3100
3101 struct dma_map_ops intel_dma_ops = {
3102         .alloc_coherent = intel_alloc_coherent,
3103         .free_coherent = intel_free_coherent,
3104         .map_sg = intel_map_sg,
3105         .unmap_sg = intel_unmap_sg,
3106         .map_page = intel_map_page,
3107         .unmap_page = intel_unmap_page,
3108         .mapping_error = intel_mapping_error,
3109 };
3110
3111 static inline int iommu_domain_cache_init(void)
3112 {
3113         int ret = 0;
3114
3115         iommu_domain_cache = kmem_cache_create("iommu_domain",
3116                                          sizeof(struct dmar_domain),
3117                                          0,
3118                                          SLAB_HWCACHE_ALIGN,
3119
3120                                          NULL);
3121         if (!iommu_domain_cache) {
3122                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3123                 ret = -ENOMEM;
3124         }
3125
3126         return ret;
3127 }
3128
3129 static inline int iommu_devinfo_cache_init(void)
3130 {
3131         int ret = 0;
3132
3133         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3134                                          sizeof(struct device_domain_info),
3135                                          0,
3136                                          SLAB_HWCACHE_ALIGN,
3137                                          NULL);
3138         if (!iommu_devinfo_cache) {
3139                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3140                 ret = -ENOMEM;
3141         }
3142
3143         return ret;
3144 }
3145
3146 static inline int iommu_iova_cache_init(void)
3147 {
3148         int ret = 0;
3149
3150         iommu_iova_cache = kmem_cache_create("iommu_iova",
3151                                          sizeof(struct iova),
3152                                          0,
3153                                          SLAB_HWCACHE_ALIGN,
3154                                          NULL);
3155         if (!iommu_iova_cache) {
3156                 printk(KERN_ERR "Couldn't create iova cache\n");
3157                 ret = -ENOMEM;
3158         }
3159
3160         return ret;
3161 }
3162
3163 static int __init iommu_init_mempool(void)
3164 {
3165         int ret;
3166         ret = iommu_iova_cache_init();
3167         if (ret)
3168                 return ret;
3169
3170         ret = iommu_domain_cache_init();
3171         if (ret)
3172                 goto domain_error;
3173
3174         ret = iommu_devinfo_cache_init();
3175         if (!ret)
3176                 return ret;
3177
3178         kmem_cache_destroy(iommu_domain_cache);
3179 domain_error:
3180         kmem_cache_destroy(iommu_iova_cache);
3181
3182         return -ENOMEM;
3183 }
3184
3185 static void __init iommu_exit_mempool(void)
3186 {
3187         kmem_cache_destroy(iommu_devinfo_cache);
3188         kmem_cache_destroy(iommu_domain_cache);
3189         kmem_cache_destroy(iommu_iova_cache);
3190
3191 }
3192
3193 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3194 {
3195         struct dmar_drhd_unit *drhd;
3196         u32 vtbar;
3197         int rc;
3198
3199         /* We know that this device on this chipset has its own IOMMU.
3200          * If we find it under a different IOMMU, then the BIOS is lying
3201          * to us. Hope that the IOMMU for this device is actually
3202          * disabled, and it needs no translation...
3203          */
3204         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3205         if (rc) {
3206                 /* "can't" happen */
3207                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3208                 return;
3209         }
3210         vtbar &= 0xffff0000;
3211
3212         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3213         drhd = dmar_find_matched_drhd_unit(pdev);
3214         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3215                             TAINT_FIRMWARE_WORKAROUND,
3216                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3217                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3218 }
3219 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3220
3221 static void __init init_no_remapping_devices(void)
3222 {
3223         struct dmar_drhd_unit *drhd;
3224
3225         for_each_drhd_unit(drhd) {
3226                 if (!drhd->include_all) {
3227                         int i;
3228                         for (i = 0; i < drhd->devices_cnt; i++)
3229                                 if (drhd->devices[i] != NULL)
3230                                         break;
3231                         /* ignore DMAR unit if no pci devices exist */
3232                         if (i == drhd->devices_cnt)
3233                                 drhd->ignored = 1;
3234                 }
3235         }
3236
3237         for_each_drhd_unit(drhd) {
3238                 int i;
3239                 if (drhd->ignored || drhd->include_all)
3240                         continue;
3241
3242                 for (i = 0; i < drhd->devices_cnt; i++)
3243                         if (drhd->devices[i] &&
3244                             !IS_GFX_DEVICE(drhd->devices[i]))
3245                                 break;
3246
3247                 if (i < drhd->devices_cnt)
3248                         continue;
3249
3250                 /* This IOMMU has *only* gfx devices. Either bypass it or
3251                    set the gfx_mapped flag, as appropriate */
3252                 if (dmar_map_gfx) {
3253                         intel_iommu_gfx_mapped = 1;
3254                 } else {
3255                         drhd->ignored = 1;
3256                         for (i = 0; i < drhd->devices_cnt; i++) {
3257                                 if (!drhd->devices[i])
3258                                         continue;
3259                                 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3260                         }
3261                 }
3262         }
3263 }
3264
3265 #ifdef CONFIG_SUSPEND
3266 static int init_iommu_hw(void)
3267 {
3268         struct dmar_drhd_unit *drhd;
3269         struct intel_iommu *iommu = NULL;
3270
3271         for_each_active_iommu(iommu, drhd)
3272                 if (iommu->qi)
3273                         dmar_reenable_qi(iommu);
3274
3275         for_each_iommu(iommu, drhd) {
3276                 if (drhd->ignored) {
3277                         /*
3278                          * we always have to disable PMRs or DMA may fail on
3279                          * this device
3280                          */
3281                         if (force_on)
3282                                 iommu_disable_protect_mem_regions(iommu);
3283                         continue;
3284                 }
3285
3286                 iommu_flush_write_buffer(iommu);
3287
3288                 iommu_set_root_entry(iommu);
3289
3290                 iommu->flush.flush_context(iommu, 0, 0, 0,
3291                                            DMA_CCMD_GLOBAL_INVL);
3292                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3293                                          DMA_TLB_GLOBAL_FLUSH);
3294                 if (iommu_enable_translation(iommu))
3295                         return 1;
3296                 iommu_disable_protect_mem_regions(iommu);
3297         }
3298
3299         return 0;
3300 }
3301
3302 static void iommu_flush_all(void)
3303 {
3304         struct dmar_drhd_unit *drhd;
3305         struct intel_iommu *iommu;
3306
3307         for_each_active_iommu(iommu, drhd) {
3308                 iommu->flush.flush_context(iommu, 0, 0, 0,
3309                                            DMA_CCMD_GLOBAL_INVL);
3310                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3311                                          DMA_TLB_GLOBAL_FLUSH);
3312         }
3313 }
3314
3315 static int iommu_suspend(void)
3316 {
3317         struct dmar_drhd_unit *drhd;
3318         struct intel_iommu *iommu = NULL;
3319         unsigned long flag;
3320
3321         for_each_active_iommu(iommu, drhd) {
3322                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3323                                                  GFP_ATOMIC);
3324                 if (!iommu->iommu_state)
3325                         goto nomem;
3326         }
3327
3328         iommu_flush_all();
3329
3330         for_each_active_iommu(iommu, drhd) {
3331                 iommu_disable_translation(iommu);
3332
3333                 spin_lock_irqsave(&iommu->register_lock, flag);
3334
3335                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3336                         readl(iommu->reg + DMAR_FECTL_REG);
3337                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3338                         readl(iommu->reg + DMAR_FEDATA_REG);
3339                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3340                         readl(iommu->reg + DMAR_FEADDR_REG);
3341                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3342                         readl(iommu->reg + DMAR_FEUADDR_REG);
3343
3344                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3345         }
3346         return 0;
3347
3348 nomem:
3349         for_each_active_iommu(iommu, drhd)
3350                 kfree(iommu->iommu_state);
3351
3352         return -ENOMEM;
3353 }
3354
3355 static void iommu_resume(void)
3356 {
3357         struct dmar_drhd_unit *drhd;
3358         struct intel_iommu *iommu = NULL;
3359         unsigned long flag;
3360
3361         if (init_iommu_hw()) {
3362                 if (force_on)
3363                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3364                 else
3365                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3366                 return;
3367         }
3368
3369         for_each_active_iommu(iommu, drhd) {
3370
3371                 spin_lock_irqsave(&iommu->register_lock, flag);
3372
3373                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3374                         iommu->reg + DMAR_FECTL_REG);
3375                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3376                         iommu->reg + DMAR_FEDATA_REG);
3377                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3378                         iommu->reg + DMAR_FEADDR_REG);
3379                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3380                         iommu->reg + DMAR_FEUADDR_REG);
3381
3382                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3383         }
3384
3385         for_each_active_iommu(iommu, drhd)
3386                 kfree(iommu->iommu_state);
3387 }
3388
3389 static struct syscore_ops iommu_syscore_ops = {
3390         .resume         = iommu_resume,
3391         .suspend        = iommu_suspend,
3392 };
3393
3394 static void __init init_iommu_pm_ops(void)
3395 {
3396         register_syscore_ops(&iommu_syscore_ops);
3397 }
3398
3399 #else
3400 static inline void init_iommu_pm_ops(void) {}
3401 #endif  /* CONFIG_PM */
3402
3403 /*
3404  * Here we only respond to action of unbound device from driver.
3405  *
3406  * Added device is not attached to its DMAR domain here yet. That will happen
3407  * when mapping the device to iova.
3408  */
3409 static int device_notifier(struct notifier_block *nb,
3410                                   unsigned long action, void *data)
3411 {
3412         struct device *dev = data;
3413         struct pci_dev *pdev = to_pci_dev(dev);
3414         struct dmar_domain *domain;
3415
3416         if (iommu_no_mapping(dev))
3417                 return 0;
3418
3419         domain = find_domain(pdev);
3420         if (!domain)
3421                 return 0;
3422
3423         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3424                 domain_remove_one_dev_info(domain, pdev);
3425
3426                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3427                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3428                     list_empty(&domain->devices))
3429                         domain_exit(domain);
3430         }
3431
3432         return 0;
3433 }
3434
3435 static struct notifier_block device_nb = {
3436         .notifier_call = device_notifier,
3437 };
3438
3439 int __init intel_iommu_init(void)
3440 {
3441         int ret = 0;
3442
3443         /* VT-d is required for a TXT/tboot launch, so enforce that */
3444         force_on = tboot_force_iommu();
3445
3446         if (dmar_table_init()) {
3447                 if (force_on)
3448                         panic("tboot: Failed to initialize DMAR table\n");
3449                 return  -ENODEV;
3450         }
3451
3452         if (dmar_dev_scope_init()) {
3453                 if (force_on)
3454                         panic("tboot: Failed to initialize DMAR device scope\n");
3455                 return  -ENODEV;
3456         }
3457
3458         /*
3459          * Check the need for DMA-remapping initialization now.
3460          * Above initialization will also be used by Interrupt-remapping.
3461          */
3462         if (no_iommu || dmar_disabled)
3463                 return -ENODEV;
3464
3465         if (iommu_init_mempool()) {
3466                 if (force_on)
3467                         panic("tboot: Failed to initialize iommu memory\n");
3468                 return  -ENODEV;
3469         }
3470
3471         if (dmar_init_reserved_ranges()) {
3472                 if (force_on)
3473                         panic("tboot: Failed to reserve iommu ranges\n");
3474                 return  -ENODEV;
3475         }
3476
3477         init_no_remapping_devices();
3478
3479         ret = init_dmars();
3480         if (ret) {
3481                 if (force_on)
3482                         panic("tboot: Failed to initialize DMARs\n");
3483                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3484                 put_iova_domain(&reserved_iova_list);
3485                 iommu_exit_mempool();
3486                 return ret;
3487         }
3488         printk(KERN_INFO
3489         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3490
3491         init_timer(&unmap_timer);
3492 #ifdef CONFIG_SWIOTLB
3493         swiotlb = 0;
3494 #endif
3495         dma_ops = &intel_dma_ops;
3496
3497         init_iommu_pm_ops();
3498
3499         register_iommu(&intel_iommu_ops);
3500
3501         bus_register_notifier(&pci_bus_type, &device_nb);
3502
3503         return 0;
3504 }
3505
3506 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3507                                            struct pci_dev *pdev)
3508 {
3509         struct pci_dev *tmp, *parent;
3510
3511         if (!iommu || !pdev)
3512                 return;
3513
3514         /* dependent device detach */
3515         tmp = pci_find_upstream_pcie_bridge(pdev);
3516         /* Secondary interface's bus number and devfn 0 */
3517         if (tmp) {
3518                 parent = pdev->bus->self;
3519                 while (parent != tmp) {
3520                         iommu_detach_dev(iommu, parent->bus->number,
3521                                          parent->devfn);
3522                         parent = parent->bus->self;
3523                 }
3524                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3525                         iommu_detach_dev(iommu,
3526                                 tmp->subordinate->number, 0);
3527                 else /* this is a legacy PCI bridge */
3528                         iommu_detach_dev(iommu, tmp->bus->number,
3529                                          tmp->devfn);
3530         }
3531 }
3532
3533 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3534                                           struct pci_dev *pdev)
3535 {
3536         struct device_domain_info *info;
3537         struct intel_iommu *iommu;
3538         unsigned long flags;
3539         int found = 0;
3540         struct list_head *entry, *tmp;
3541
3542         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3543                                 pdev->devfn);
3544         if (!iommu)
3545                 return;
3546
3547         spin_lock_irqsave(&device_domain_lock, flags);
3548         list_for_each_safe(entry, tmp, &domain->devices) {
3549                 info = list_entry(entry, struct device_domain_info, link);
3550                 if (info->segment == pci_domain_nr(pdev->bus) &&
3551                     info->bus == pdev->bus->number &&
3552                     info->devfn == pdev->devfn) {
3553                         list_del(&info->link);
3554                         list_del(&info->global);
3555                         if (info->dev)
3556                                 info->dev->dev.archdata.iommu = NULL;
3557                         spin_unlock_irqrestore(&device_domain_lock, flags);
3558
3559                         iommu_disable_dev_iotlb(info);
3560                         iommu_detach_dev(iommu, info->bus, info->devfn);
3561                         iommu_detach_dependent_devices(iommu, pdev);
3562                         free_devinfo_mem(info);
3563
3564                         spin_lock_irqsave(&device_domain_lock, flags);
3565
3566                         if (found)
3567                                 break;
3568                         else
3569                                 continue;
3570                 }
3571
3572                 /* if there is no other devices under the same iommu
3573                  * owned by this domain, clear this iommu in iommu_bmp
3574                  * update iommu count and coherency
3575                  */
3576                 if (iommu == device_to_iommu(info->segment, info->bus,
3577                                             info->devfn))
3578                         found = 1;
3579         }
3580
3581         spin_unlock_irqrestore(&device_domain_lock, flags);
3582
3583         if (found == 0) {
3584                 unsigned long tmp_flags;
3585                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3586                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3587                 domain->iommu_count--;
3588                 domain_update_iommu_cap(domain);
3589                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3590
3591                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3592                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3593                         spin_lock_irqsave(&iommu->lock, tmp_flags);
3594                         clear_bit(domain->id, iommu->domain_ids);
3595                         iommu->domains[domain->id] = NULL;
3596                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3597                 }
3598         }
3599 }
3600
3601 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3602 {
3603         struct device_domain_info *info;
3604         struct intel_iommu *iommu;
3605         unsigned long flags1, flags2;
3606
3607         spin_lock_irqsave(&device_domain_lock, flags1);
3608         while (!list_empty(&domain->devices)) {
3609                 info = list_entry(domain->devices.next,
3610                         struct device_domain_info, link);
3611                 list_del(&info->link);
3612                 list_del(&info->global);
3613                 if (info->dev)
3614                         info->dev->dev.archdata.iommu = NULL;
3615
3616                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3617
3618                 iommu_disable_dev_iotlb(info);
3619                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3620                 iommu_detach_dev(iommu, info->bus, info->devfn);
3621                 iommu_detach_dependent_devices(iommu, info->dev);
3622
3623                 /* clear this iommu in iommu_bmp, update iommu count
3624                  * and capabilities
3625                  */
3626                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3627                 if (test_and_clear_bit(iommu->seq_id,
3628                                        &domain->iommu_bmp)) {
3629                         domain->iommu_count--;
3630                         domain_update_iommu_cap(domain);
3631                 }
3632                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3633
3634                 free_devinfo_mem(info);
3635                 spin_lock_irqsave(&device_domain_lock, flags1);
3636         }
3637         spin_unlock_irqrestore(&device_domain_lock, flags1);
3638 }
3639
3640 /* domain id for virtual machine, it won't be set in context */
3641 static unsigned long vm_domid;
3642
3643 static struct dmar_domain *iommu_alloc_vm_domain(void)
3644 {
3645         struct dmar_domain *domain;
3646
3647         domain = alloc_domain_mem();
3648         if (!domain)
3649                 return NULL;
3650
3651         domain->id = vm_domid++;
3652         domain->nid = -1;
3653         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3654         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3655
3656         return domain;
3657 }
3658
3659 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3660 {
3661         int adjust_width;
3662
3663         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3664         spin_lock_init(&domain->iommu_lock);
3665
3666         domain_reserve_special_ranges(domain);
3667
3668         /* calculate AGAW */
3669         domain->gaw = guest_width;
3670         adjust_width = guestwidth_to_adjustwidth(guest_width);
3671         domain->agaw = width_to_agaw(adjust_width);
3672
3673         INIT_LIST_HEAD(&domain->devices);
3674
3675         domain->iommu_count = 0;
3676         domain->iommu_coherency = 0;
3677         domain->iommu_snooping = 0;
3678         domain->iommu_superpage = 0;
3679         domain->max_addr = 0;
3680         domain->nid = -1;
3681
3682         /* always allocate the top pgd */
3683         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3684         if (!domain->pgd)
3685                 return -ENOMEM;
3686         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3687         return 0;
3688 }
3689
3690 static void iommu_free_vm_domain(struct dmar_domain *domain)
3691 {
3692         unsigned long flags;
3693         struct dmar_drhd_unit *drhd;
3694         struct intel_iommu *iommu;
3695         unsigned long i;
3696         unsigned long ndomains;
3697
3698         for_each_drhd_unit(drhd) {
3699                 if (drhd->ignored)
3700                         continue;
3701                 iommu = drhd->iommu;
3702
3703                 ndomains = cap_ndoms(iommu->cap);
3704                 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3705                         if (iommu->domains[i] == domain) {
3706                                 spin_lock_irqsave(&iommu->lock, flags);
3707                                 clear_bit(i, iommu->domain_ids);
3708                                 iommu->domains[i] = NULL;
3709                                 spin_unlock_irqrestore(&iommu->lock, flags);
3710                                 break;
3711                         }
3712                 }
3713         }
3714 }
3715
3716 static void vm_domain_exit(struct dmar_domain *domain)
3717 {
3718         /* Domain 0 is reserved, so dont process it */
3719         if (!domain)
3720                 return;
3721
3722         vm_domain_remove_all_dev_info(domain);
3723         /* destroy iovas */
3724         put_iova_domain(&domain->iovad);
3725
3726         /* clear ptes */
3727         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3728
3729         /* free page tables */
3730         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3731
3732         iommu_free_vm_domain(domain);
3733         free_domain_mem(domain);
3734 }
3735
3736 static int intel_iommu_domain_init(struct iommu_domain *domain)
3737 {
3738         struct dmar_domain *dmar_domain;
3739
3740         dmar_domain = iommu_alloc_vm_domain();
3741         if (!dmar_domain) {
3742                 printk(KERN_ERR
3743                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3744                 return -ENOMEM;
3745         }
3746         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3747                 printk(KERN_ERR
3748                         "intel_iommu_domain_init() failed\n");
3749                 vm_domain_exit(dmar_domain);
3750                 return -ENOMEM;
3751         }
3752         domain_update_iommu_cap(dmar_domain);
3753         domain->priv = dmar_domain;
3754
3755         return 0;
3756 }
3757
3758 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3759 {
3760         struct dmar_domain *dmar_domain = domain->priv;
3761
3762         domain->priv = NULL;
3763         vm_domain_exit(dmar_domain);
3764 }
3765
3766 static int intel_iommu_attach_device(struct iommu_domain *domain,
3767                                      struct device *dev)
3768 {
3769         struct dmar_domain *dmar_domain = domain->priv;
3770         struct pci_dev *pdev = to_pci_dev(dev);
3771         struct intel_iommu *iommu;
3772         int addr_width;
3773
3774         /* normally pdev is not mapped */
3775         if (unlikely(domain_context_mapped(pdev))) {
3776                 struct dmar_domain *old_domain;
3777
3778                 old_domain = find_domain(pdev);
3779                 if (old_domain) {
3780                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3781                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3782                                 domain_remove_one_dev_info(old_domain, pdev);
3783                         else
3784                                 domain_remove_dev_info(old_domain);
3785                 }
3786         }
3787
3788         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3789                                 pdev->devfn);
3790         if (!iommu)
3791                 return -ENODEV;
3792
3793         /* check if this iommu agaw is sufficient for max mapped address */
3794         addr_width = agaw_to_width(iommu->agaw);
3795         if (addr_width > cap_mgaw(iommu->cap))
3796                 addr_width = cap_mgaw(iommu->cap);
3797
3798         if (dmar_domain->max_addr > (1LL << addr_width)) {
3799                 printk(KERN_ERR "%s: iommu width (%d) is not "
3800                        "sufficient for the mapped address (%llx)\n",
3801                        __func__, addr_width, dmar_domain->max_addr);
3802                 return -EFAULT;
3803         }
3804         dmar_domain->gaw = addr_width;
3805
3806         /*
3807          * Knock out extra levels of page tables if necessary
3808          */
3809         while (iommu->agaw < dmar_domain->agaw) {
3810                 struct dma_pte *pte;
3811
3812                 pte = dmar_domain->pgd;
3813                 if (dma_pte_present(pte)) {
3814                         dmar_domain->pgd = (struct dma_pte *)
3815                                 phys_to_virt(dma_pte_addr(pte));
3816                         free_pgtable_page(pte);
3817                 }
3818                 dmar_domain->agaw--;
3819         }
3820
3821         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3822 }
3823
3824 static void intel_iommu_detach_device(struct iommu_domain *domain,
3825                                       struct device *dev)
3826 {
3827         struct dmar_domain *dmar_domain = domain->priv;
3828         struct pci_dev *pdev = to_pci_dev(dev);
3829
3830         domain_remove_one_dev_info(dmar_domain, pdev);
3831 }
3832
3833 static int intel_iommu_map(struct iommu_domain *domain,
3834                            unsigned long iova, phys_addr_t hpa,
3835                            int gfp_order, int iommu_prot)
3836 {
3837         struct dmar_domain *dmar_domain = domain->priv;
3838         u64 max_addr;
3839         int prot = 0;
3840         size_t size;
3841         int ret;
3842
3843         if (iommu_prot & IOMMU_READ)
3844                 prot |= DMA_PTE_READ;
3845         if (iommu_prot & IOMMU_WRITE)
3846                 prot |= DMA_PTE_WRITE;
3847         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3848                 prot |= DMA_PTE_SNP;
3849
3850         size     = PAGE_SIZE << gfp_order;
3851         max_addr = iova + size;
3852         if (dmar_domain->max_addr < max_addr) {
3853                 u64 end;
3854
3855                 /* check if minimum agaw is sufficient for mapped address */
3856                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3857                 if (end < max_addr) {
3858                         printk(KERN_ERR "%s: iommu width (%d) is not "
3859                                "sufficient for the mapped address (%llx)\n",
3860                                __func__, dmar_domain->gaw, max_addr);
3861                         return -EFAULT;
3862                 }
3863                 dmar_domain->max_addr = max_addr;
3864         }
3865         /* Round up size to next multiple of PAGE_SIZE, if it and
3866            the low bits of hpa would take us onto the next page */
3867         size = aligned_nrpages(hpa, size);
3868         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3869                                  hpa >> VTD_PAGE_SHIFT, size, prot);
3870         return ret;
3871 }
3872
3873 static int intel_iommu_unmap(struct iommu_domain *domain,
3874                              unsigned long iova, int gfp_order)
3875 {
3876         struct dmar_domain *dmar_domain = domain->priv;
3877         size_t size = PAGE_SIZE << gfp_order;
3878         int order;
3879
3880         order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3881                             (iova + size - 1) >> VTD_PAGE_SHIFT);
3882
3883         if (dmar_domain->max_addr == iova + size)
3884                 dmar_domain->max_addr = iova;
3885
3886         return order;
3887 }
3888
3889 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3890                                             unsigned long iova)
3891 {
3892         struct dmar_domain *dmar_domain = domain->priv;
3893         struct dma_pte *pte;
3894         u64 phys = 0;
3895
3896         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
3897         if (pte)
3898                 phys = dma_pte_addr(pte);
3899
3900         return phys;
3901 }
3902
3903 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3904                                       unsigned long cap)
3905 {
3906         struct dmar_domain *dmar_domain = domain->priv;
3907
3908         if (cap == IOMMU_CAP_CACHE_COHERENCY)
3909                 return dmar_domain->iommu_snooping;
3910         if (cap == IOMMU_CAP_INTR_REMAP)
3911                 return intr_remapping_enabled;
3912
3913         return 0;
3914 }
3915
3916 static struct iommu_ops intel_iommu_ops = {
3917         .domain_init    = intel_iommu_domain_init,
3918         .domain_destroy = intel_iommu_domain_destroy,
3919         .attach_dev     = intel_iommu_attach_device,
3920         .detach_dev     = intel_iommu_detach_device,
3921         .map            = intel_iommu_map,
3922         .unmap          = intel_iommu_unmap,
3923         .iova_to_phys   = intel_iommu_iova_to_phys,
3924         .domain_has_cap = intel_iommu_domain_has_cap,
3925 };
3926
3927 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3928 {
3929         /*
3930          * Mobile 4 Series Chipset neglects to set RWBF capability,
3931          * but needs it:
3932          */
3933         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3934         rwbf_quirk = 1;
3935
3936         /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3937         if (dev->revision == 0x07) {
3938                 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3939                 dmar_map_gfx = 0;
3940         }
3941 }
3942
3943 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3944
3945 #define GGC 0x52
3946 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
3947 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
3948 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
3949 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
3950 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
3951 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
3952 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
3953 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
3954
3955 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3956 {
3957         unsigned short ggc;
3958
3959         if (pci_read_config_word(dev, GGC, &ggc))
3960                 return;
3961
3962         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3963                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3964                 dmar_map_gfx = 0;
3965         } else if (dmar_map_gfx) {
3966                 /* we have to ensure the gfx device is idle before we flush */
3967                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
3968                 intel_iommu_strict = 1;
3969        }
3970 }
3971 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3972 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3973 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3974 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3975
3976 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3977    ISOCH DMAR unit for the Azalia sound device, but not give it any
3978    TLB entries, which causes it to deadlock. Check for that.  We do
3979    this in a function called from init_dmars(), instead of in a PCI
3980    quirk, because we don't want to print the obnoxious "BIOS broken"
3981    message if VT-d is actually disabled.
3982 */
3983 static void __init check_tylersburg_isoch(void)
3984 {
3985         struct pci_dev *pdev;
3986         uint32_t vtisochctrl;
3987
3988         /* If there's no Azalia in the system anyway, forget it. */
3989         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3990         if (!pdev)
3991                 return;
3992         pci_dev_put(pdev);
3993
3994         /* System Management Registers. Might be hidden, in which case
3995            we can't do the sanity check. But that's OK, because the
3996            known-broken BIOSes _don't_ actually hide it, so far. */
3997         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3998         if (!pdev)
3999                 return;
4000
4001         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4002                 pci_dev_put(pdev);
4003                 return;
4004         }
4005
4006         pci_dev_put(pdev);
4007
4008         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4009         if (vtisochctrl & 1)
4010                 return;
4011
4012         /* Drop all bits other than the number of TLB entries */
4013         vtisochctrl &= 0x1c;
4014
4015         /* If we have the recommended number of TLB entries (16), fine. */
4016         if (vtisochctrl == 0x10)
4017                 return;
4018
4019         /* Zero TLB entries? You get to ride the short bus to school. */
4020         if (!vtisochctrl) {
4021                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4022                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4023                      dmi_get_system_info(DMI_BIOS_VENDOR),
4024                      dmi_get_system_info(DMI_BIOS_VERSION),
4025                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4026                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4027                 return;
4028         }
4029
4030         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4031                vtisochctrl);
4032 }