drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/delay.h>
  32 #include <linux/dma-iommu.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/err.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/io.h>
  37 #include <linux/io-64-nonatomic-hi-lo.h>
  38 #include <linux/iommu.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/module.h>
  41 #include <linux/of.h>
  42 #include <linux/of_address.h>
  43 #include <linux/pci.h>
  44 #include <linux/platform_device.h>
  45 #include <linux/slab.h>
  46 #include <linux/spinlock.h>
  47
  48 #include <linux/amba/bus.h>
  49
  50 #include "io-pgtable.h"
  51
  52 /* Maximum number of stream IDs assigned to a single device */
  53 #define MAX_MASTER_STREAMIDS            MAX_PHANDLE_ARGS
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* Maximum number of mapping groups per SMMU */
  59 #define ARM_SMMU_MAX_SMRS               128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_GCFGFRE                    (1 << 4)
  92 #define sCR0_GCFGFIE                    (1 << 5)
  93 #define sCR0_USFCFG                     (1 << 10)
  94 #define sCR0_VMIDPNE                    (1 << 11)
  95 #define sCR0_PTM                        (1 << 12)
  96 #define sCR0_FB                         (1 << 13)
  97 #define sCR0_VMID16EN                   (1 << 31)
  98 #define sCR0_BSU_SHIFT                  14
  99 #define sCR0_BSU_MASK                   0x3
 100
 101 /* Auxiliary Configuration register */
 102 #define ARM_SMMU_GR0_sACR               0x10
 103
 104 /* Identification registers */
 105 #define ARM_SMMU_GR0_ID0                0x20
 106 #define ARM_SMMU_GR0_ID1                0x24
 107 #define ARM_SMMU_GR0_ID2                0x28
 108 #define ARM_SMMU_GR0_ID3                0x2c
 109 #define ARM_SMMU_GR0_ID4                0x30
 110 #define ARM_SMMU_GR0_ID5                0x34
 111 #define ARM_SMMU_GR0_ID6                0x38
 112 #define ARM_SMMU_GR0_ID7                0x3c
 113 #define ARM_SMMU_GR0_sGFSR              0x48
 114 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 115 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 116 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 117
 118 #define ID0_S1TS                        (1 << 30)
 119 #define ID0_S2TS                        (1 << 29)
 120 #define ID0_NTS                         (1 << 28)
 121 #define ID0_SMS                         (1 << 27)
 122 #define ID0_ATOSNS                      (1 << 26)
 123 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 124 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 125 #define ID0_CTTW                        (1 << 14)
 126 #define ID0_NUMIRPT_SHIFT               16
 127 #define ID0_NUMIRPT_MASK                0xff
 128 #define ID0_NUMSIDB_SHIFT               9
 129 #define ID0_NUMSIDB_MASK                0xf
 130 #define ID0_NUMSMRG_SHIFT               0
 131 #define ID0_NUMSMRG_MASK                0xff
 132
 133 #define ID1_PAGESIZE                    (1 << 31)
 134 #define ID1_NUMPAGENDXB_SHIFT           28
 135 #define ID1_NUMPAGENDXB_MASK            7
 136 #define ID1_NUMS2CB_SHIFT               16
 137 #define ID1_NUMS2CB_MASK                0xff
 138 #define ID1_NUMCB_SHIFT                 0
 139 #define ID1_NUMCB_MASK                  0xff
 140
 141 #define ID2_OAS_SHIFT                   4
 142 #define ID2_OAS_MASK                    0xf
 143 #define ID2_IAS_SHIFT                   0
 144 #define ID2_IAS_MASK                    0xf
 145 #define ID2_UBS_SHIFT                   8
 146 #define ID2_UBS_MASK                    0xf
 147 #define ID2_PTFS_4K                     (1 << 12)
 148 #define ID2_PTFS_16K                    (1 << 13)
 149 #define ID2_PTFS_64K                    (1 << 14)
 150 #define ID2_VMID16                      (1 << 15)
 151
 152 #define ID7_MAJOR_SHIFT                 4
 153 #define ID7_MAJOR_MASK                  0xf
 154
 155 /* Global TLB invalidation */
 156 #define ARM_SMMU_GR0_TLBIVMID           0x64
 157 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 158 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 159 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 160 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 161 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 162 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 163
 164 /* Stream mapping registers */
 165 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 166 #define SMR_VALID                       (1 << 31)
 167 #define SMR_MASK_SHIFT                  16
 168 #define SMR_MASK_MASK                   0x7fff
 169 #define SMR_ID_SHIFT                    0
 170 #define SMR_ID_MASK                     0x7fff
 171
 172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 173 #define S2CR_CBNDX_SHIFT                0
 174 #define S2CR_CBNDX_MASK                 0xff
 175 #define S2CR_TYPE_SHIFT                 16
 176 #define S2CR_TYPE_MASK                  0x3
 177 #define S2CR_TYPE_TRANS                 (0 << S2CR_TYPE_SHIFT)
 178 #define S2CR_TYPE_BYPASS                (1 << S2CR_TYPE_SHIFT)
 179 #define S2CR_TYPE_FAULT                 (2 << S2CR_TYPE_SHIFT)
 180
 181 #define S2CR_PRIVCFG_SHIFT              24
 182 #define S2CR_PRIVCFG_UNPRIV             (2 << S2CR_PRIVCFG_SHIFT)
 183
 184 /* Context bank attribute registers */
 185 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 186 #define CBAR_VMID_SHIFT                 0
 187 #define CBAR_VMID_MASK                  0xff
 188 #define CBAR_S1_BPSHCFG_SHIFT           8
 189 #define CBAR_S1_BPSHCFG_MASK            3
 190 #define CBAR_S1_BPSHCFG_NSH             3
 191 #define CBAR_S1_MEMATTR_SHIFT           12
 192 #define CBAR_S1_MEMATTR_MASK            0xf
 193 #define CBAR_S1_MEMATTR_WB              0xf
 194 #define CBAR_TYPE_SHIFT                 16
 195 #define CBAR_TYPE_MASK                  0x3
 196 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 197 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 198 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 199 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 200 #define CBAR_IRPTNDX_SHIFT              24
 201 #define CBAR_IRPTNDX_MASK               0xff
 202
 203 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 204 #define CBA2R_RW64_32BIT                (0 << 0)
 205 #define CBA2R_RW64_64BIT                (1 << 0)
 206 #define CBA2R_VMID_SHIFT                16
 207 #define CBA2R_VMID_MASK                 0xffff
 208
 209 /* Translation context bank */
 210 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 211 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 212
 213 #define ARM_SMMU_CB_SCTLR               0x0
 214 #define ARM_SMMU_CB_ACTLR               0x4
 215 #define ARM_SMMU_CB_RESUME              0x8
 216 #define ARM_SMMU_CB_TTBCR2              0x10
 217 #define ARM_SMMU_CB_TTBR0               0x20
 218 #define ARM_SMMU_CB_TTBR1               0x28
 219 #define ARM_SMMU_CB_TTBCR               0x30
 220 #define ARM_SMMU_CB_S1_MAIR0            0x38
 221 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 222 #define ARM_SMMU_CB_PAR                 0x50
 223 #define ARM_SMMU_CB_FSR                 0x58
 224 #define ARM_SMMU_CB_FAR                 0x60
 225 #define ARM_SMMU_CB_FSYNR0              0x68
 226 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 227 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 228 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 229 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 230 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 231 #define ARM_SMMU_CB_ATS1PR              0x800
 232 #define ARM_SMMU_CB_ATSR                0x8f0
 233
 234 #define SCTLR_S1_ASIDPNE                (1 << 12)
 235 #define SCTLR_CFCFG                     (1 << 7)
 236 #define SCTLR_CFIE                      (1 << 6)
 237 #define SCTLR_CFRE                      (1 << 5)
 238 #define SCTLR_E                         (1 << 4)
 239 #define SCTLR_AFE                       (1 << 2)
 240 #define SCTLR_TRE                       (1 << 1)
 241 #define SCTLR_M                         (1 << 0)
 242 #define SCTLR_EAE_SBOP                  (SCTLR_AFE | SCTLR_TRE)
 243
 244 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 245
 246 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 247
 248 #define CB_PAR_F                        (1 << 0)
 249
 250 #define ATSR_ACTIVE                     (1 << 0)
 251
 252 #define RESUME_RETRY                    (0 << 0)
 253 #define RESUME_TERMINATE                (1 << 0)
 254
 255 #define TTBCR2_SEP_SHIFT                15
 256 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 257
 258 #define TTBRn_ASID_SHIFT                48
 259
 260 #define FSR_MULTI                       (1 << 31)
 261 #define FSR_SS                          (1 << 30)
 262 #define FSR_UUT                         (1 << 8)
 263 #define FSR_ASF                         (1 << 7)
 264 #define FSR_TLBLKF                      (1 << 6)
 265 #define FSR_TLBMCF                      (1 << 5)
 266 #define FSR_EF                          (1 << 4)
 267 #define FSR_PF                          (1 << 3)
 268 #define FSR_AFF                         (1 << 2)
 269 #define FSR_TF                          (1 << 1)
 270
 271 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 272                                          FSR_TLBMCF | FSR_TLBLKF)
 273 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 274                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 275
 276 #define FSYNR0_WNR                      (1 << 4)
 277
 278 static int force_stage;
 279 module_param(force_stage, int, S_IRUGO);
 280 MODULE_PARM_DESC(force_stage,
 281         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 282 static bool disable_bypass;
 283 module_param(disable_bypass, bool, S_IRUGO);
 284 MODULE_PARM_DESC(disable_bypass,
 285         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 286
 287 enum arm_smmu_arch_version {
 288         ARM_SMMU_V1,
 289         ARM_SMMU_V1_64K,
 290         ARM_SMMU_V2,
 291 };
 292
 293 enum arm_smmu_implementation {
 294         GENERIC_SMMU,
 295         ARM_MMU500,
 296         CAVIUM_SMMUV2,
 297 };
 298
 299 struct arm_smmu_smr {
 300         u8                              idx;
 301         u16                             mask;
 302         u16                             id;
 303 };
 304
 305 struct arm_smmu_master_cfg {
 306         int                             num_streamids;
 307         u16                             streamids[MAX_MASTER_STREAMIDS];
 308         struct arm_smmu_smr             *smrs;
 309 };
 310
 311 struct arm_smmu_master {
 312         struct device_node              *of_node;
 313         struct rb_node                  node;
 314         struct arm_smmu_master_cfg      cfg;
 315 };
 316
 317 struct arm_smmu_device {
 318         struct device                   *dev;
 319
 320         void __iomem                    *base;
 321         unsigned long                   size;
 322         unsigned long                   pgshift;
 323
 324 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 325 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 326 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 327 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 328 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 329 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 330 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 331 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 332 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 333 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 334 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 335 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 336         u32                             features;
 337
 338 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 339         u32                             options;
 340         enum arm_smmu_arch_version      version;
 341         enum arm_smmu_implementation    model;
 342
 343         u32                             num_context_banks;
 344         u32                             num_s2_context_banks;
 345         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 346         atomic_t                        irptndx;
 347
 348         u32                             num_mapping_groups;
 349         DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
 350
 351         unsigned long                   va_size;
 352         unsigned long                   ipa_size;
 353         unsigned long                   pa_size;
 354         unsigned long                   pgsize_bitmap;
 355
 356         u32                             num_global_irqs;
 357         u32                             num_context_irqs;
 358         unsigned int                    *irqs;
 359
 360         struct list_head                list;
 361         struct rb_root                  masters;
 362
 363         u32                             cavium_id_base; /* Specific to Cavium */
 364 };
 365
 366 enum arm_smmu_context_fmt {
 367         ARM_SMMU_CTX_FMT_NONE,
 368         ARM_SMMU_CTX_FMT_AARCH64,
 369         ARM_SMMU_CTX_FMT_AARCH32_L,
 370         ARM_SMMU_CTX_FMT_AARCH32_S,
 371 };
 372
 373 struct arm_smmu_cfg {
 374         u8                              cbndx;
 375         u8                              irptndx;
 376         u32                             cbar;
 377         enum arm_smmu_context_fmt       fmt;
 378 };
 379 #define INVALID_IRPTNDX                 0xff
 380
 381 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 382 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 383
 384 enum arm_smmu_domain_stage {
 385         ARM_SMMU_DOMAIN_S1 = 0,
 386         ARM_SMMU_DOMAIN_S2,
 387         ARM_SMMU_DOMAIN_NESTED,
 388 };
 389
 390 struct arm_smmu_domain {
 391         struct arm_smmu_device          *smmu;
 392         struct io_pgtable_ops           *pgtbl_ops;
 393         spinlock_t                      pgtbl_lock;
 394         struct arm_smmu_cfg             cfg;
 395         enum arm_smmu_domain_stage      stage;
 396         struct mutex                    init_mutex; /* Protects smmu pointer */
 397         struct iommu_domain             domain;
 398 };
 399
 400 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 401 static LIST_HEAD(arm_smmu_devices);
 402
 403 struct arm_smmu_option_prop {
 404         u32 opt;
 405         const char *prop;
 406 };
 407
 408 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 409
 410 static struct arm_smmu_option_prop arm_smmu_options[] = {
 411         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 412         { 0, NULL},
 413 };
 414
 415 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 416 {
 417         return container_of(dom, struct arm_smmu_domain, domain);
 418 }
 419
 420 static void parse_driver_options(struct arm_smmu_device *smmu)
 421 {
 422         int i = 0;
 423
 424         do {
 425                 if (of_property_read_bool(smmu->dev->of_node,
 426                                                 arm_smmu_options[i].prop)) {
 427                         smmu->options |= arm_smmu_options[i].opt;
 428                         dev_notice(smmu->dev, "option %s\n",
 429                                 arm_smmu_options[i].prop);
 430                 }
 431         } while (arm_smmu_options[++i].opt);
 432 }
 433
 434 static struct device_node *dev_get_dev_node(struct device *dev)
 435 {
 436         if (dev_is_pci(dev)) {
 437                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 438
 439                 while (!pci_is_root_bus(bus))
 440                         bus = bus->parent;
 441                 return bus->bridge->parent->of_node;
 442         }
 443
 444         return dev->of_node;
 445 }
 446
 447 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
 448                                                 struct device_node *dev_node)
 449 {
 450         struct rb_node *node = smmu->masters.rb_node;
 451
 452         while (node) {
 453                 struct arm_smmu_master *master;
 454
 455                 master = container_of(node, struct arm_smmu_master, node);
 456
 457                 if (dev_node < master->of_node)
 458                         node = node->rb_left;
 459                 else if (dev_node > master->of_node)
 460                         node = node->rb_right;
 461                 else
 462                         return master;
 463         }
 464
 465         return NULL;
 466 }
 467
 468 static struct arm_smmu_master_cfg *
 469 find_smmu_master_cfg(struct device *dev)
 470 {
 471         struct arm_smmu_master_cfg *cfg = NULL;
 472         struct iommu_group *group = iommu_group_get(dev);
 473
 474         if (group) {
 475                 cfg = iommu_group_get_iommudata(group);
 476                 iommu_group_put(group);
 477         }
 478
 479         return cfg;
 480 }
 481
 482 static int insert_smmu_master(struct arm_smmu_device *smmu,
 483                               struct arm_smmu_master *master)
 484 {
 485         struct rb_node **new, *parent;
 486
 487         new = &smmu->masters.rb_node;
 488         parent = NULL;
 489         while (*new) {
 490                 struct arm_smmu_master *this
 491                         = container_of(*new, struct arm_smmu_master, node);
 492
 493                 parent = *new;
 494                 if (master->of_node < this->of_node)
 495                         new = &((*new)->rb_left);
 496                 else if (master->of_node > this->of_node)
 497                         new = &((*new)->rb_right);
 498                 else
 499                         return -EEXIST;
 500         }
 501
 502         rb_link_node(&master->node, parent, new);
 503         rb_insert_color(&master->node, &smmu->masters);
 504         return 0;
 505 }
 506
 507 static int register_smmu_master(struct arm_smmu_device *smmu,
 508                                 struct device *dev,
 509                                 struct of_phandle_args *masterspec)
 510 {
 511         int i;
 512         struct arm_smmu_master *master;
 513
 514         master = find_smmu_master(smmu, masterspec->np);
 515         if (master) {
 516                 dev_err(dev,
 517                         "rejecting multiple registrations for master device %s\n",
 518                         masterspec->np->name);
 519                 return -EBUSY;
 520         }
 521
 522         if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
 523                 dev_err(dev,
 524                         "reached maximum number (%d) of stream IDs for master device %s\n",
 525                         MAX_MASTER_STREAMIDS, masterspec->np->name);
 526                 return -ENOSPC;
 527         }
 528
 529         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
 530         if (!master)
 531                 return -ENOMEM;
 532
 533         master->of_node                 = masterspec->np;
 534         master->cfg.num_streamids       = masterspec->args_count;
 535
 536         for (i = 0; i < master->cfg.num_streamids; ++i) {
 537                 u16 streamid = masterspec->args[i];
 538
 539                 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
 540                      (streamid >= smmu->num_mapping_groups)) {
 541                         dev_err(dev,
 542                                 "stream ID for master device %s greater than maximum allowed (%d)\n",
 543                                 masterspec->np->name, smmu->num_mapping_groups);
 544                         return -ERANGE;
 545                 }
 546                 master->cfg.streamids[i] = streamid;
 547         }
 548         return insert_smmu_master(smmu, master);
 549 }
 550
 551 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
 552 {
 553         struct arm_smmu_device *smmu;
 554         struct arm_smmu_master *master = NULL;
 555         struct device_node *dev_node = dev_get_dev_node(dev);
 556
 557         spin_lock(&arm_smmu_devices_lock);
 558         list_for_each_entry(smmu, &arm_smmu_devices, list) {
 559                 master = find_smmu_master(smmu, dev_node);
 560                 if (master)
 561                         break;
 562         }
 563         spin_unlock(&arm_smmu_devices_lock);
 564
 565         return master ? smmu : NULL;
 566 }
 567
 568 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 569 {
 570         int idx;
 571
 572         do {
 573                 idx = find_next_zero_bit(map, end, start);
 574                 if (idx == end)
 575                         return -ENOSPC;
 576         } while (test_and_set_bit(idx, map));
 577
 578         return idx;
 579 }
 580
 581 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 582 {
 583         clear_bit(idx, map);
 584 }
 585
 586 /* Wait for any pending TLB invalidations to complete */
 587 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 588 {
 589         int count = 0;
 590         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 591
 592         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 593         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 594                & sTLBGSTATUS_GSACTIVE) {
 595                 cpu_relax();
 596                 if (++count == TLB_LOOP_TIMEOUT) {
 597                         dev_err_ratelimited(smmu->dev,
 598                         "TLB sync timed out -- SMMU may be deadlocked\n");
 599                         return;
 600                 }
 601                 udelay(1);
 602         }
 603 }
 604
 605 static void arm_smmu_tlb_sync(void *cookie)
 606 {
 607         struct arm_smmu_domain *smmu_domain = cookie;
 608         __arm_smmu_tlb_sync(smmu_domain->smmu);
 609 }
 610
 611 static void arm_smmu_tlb_inv_context(void *cookie)
 612 {
 613         struct arm_smmu_domain *smmu_domain = cookie;
 614         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 615         struct arm_smmu_device *smmu = smmu_domain->smmu;
 616         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 617         void __iomem *base;
 618
 619         if (stage1) {
 620                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 621                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 622                                base + ARM_SMMU_CB_S1_TLBIASID);
 623         } else {
 624                 base = ARM_SMMU_GR0(smmu);
 625                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 626                                base + ARM_SMMU_GR0_TLBIVMID);
 627         }
 628
 629         __arm_smmu_tlb_sync(smmu);
 630 }
 631
 632 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 633                                           size_t granule, bool leaf, void *cookie)
 634 {
 635         struct arm_smmu_domain *smmu_domain = cookie;
 636         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 637         struct arm_smmu_device *smmu = smmu_domain->smmu;
 638         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 639         void __iomem *reg;
 640
 641         if (stage1) {
 642                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 643                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 644
 645                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 646                         iova &= ~12UL;
 647                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 648                         do {
 649                                 writel_relaxed(iova, reg);
 650                                 iova += granule;
 651                         } while (size -= granule);
 652                 } else {
 653                         iova >>= 12;
 654                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 655                         do {
 656                                 writeq_relaxed(iova, reg);
 657                                 iova += granule >> 12;
 658                         } while (size -= granule);
 659                 }
 660         } else if (smmu->version == ARM_SMMU_V2) {
 661                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 662                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 663                               ARM_SMMU_CB_S2_TLBIIPAS2;
 664                 iova >>= 12;
 665                 do {
 666                         smmu_write_atomic_lq(iova, reg);
 667                         iova += granule >> 12;
 668                 } while (size -= granule);
 669         } else {
 670                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 671                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 672         }
 673 }
 674
 675 static struct iommu_gather_ops arm_smmu_gather_ops = {
 676         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 677         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 678         .tlb_sync       = arm_smmu_tlb_sync,
 679 };
 680
 681 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 682 {
 683         int flags, ret;
 684         u32 fsr, fsynr, resume;
 685         unsigned long iova;
 686         struct iommu_domain *domain = dev;
 687         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 688         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 689         struct arm_smmu_device *smmu = smmu_domain->smmu;
 690         void __iomem *cb_base;
 691
 692         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 693         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 694
 695         if (!(fsr & FSR_FAULT))
 696                 return IRQ_NONE;
 697
 698         if (fsr & FSR_IGN)
 699                 dev_err_ratelimited(smmu->dev,
 700                                     "Unexpected context fault (fsr 0x%x)\n",
 701                                     fsr);
 702
 703         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 704         flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
 705
 706         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 707         if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
 708                 ret = IRQ_HANDLED;
 709                 resume = RESUME_RETRY;
 710         } else {
 711                 dev_err_ratelimited(smmu->dev,
 712                     "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 713                     iova, fsynr, cfg->cbndx);
 714                 ret = IRQ_NONE;
 715                 resume = RESUME_TERMINATE;
 716         }
 717
 718         /* Clear the faulting FSR */
 719         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 720
 721         /* Retry or terminate any stalled transactions */
 722         if (fsr & FSR_SS)
 723                 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
 724
 725         return ret;
 726 }
 727
 728 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 729 {
 730         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 731         struct arm_smmu_device *smmu = dev;
 732         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 733
 734         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 735         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 736         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 737         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 738
 739         if (!gfsr)
 740                 return IRQ_NONE;
 741
 742         dev_err_ratelimited(smmu->dev,
 743                 "Unexpected global fault, this could be serious\n");
 744         dev_err_ratelimited(smmu->dev,
 745                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 746                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 747
 748         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 749         return IRQ_HANDLED;
 750 }
 751
 752 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 753                                        struct io_pgtable_cfg *pgtbl_cfg)
 754 {
 755         u32 reg;
 756         u64 reg64;
 757         bool stage1;
 758         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 759         struct arm_smmu_device *smmu = smmu_domain->smmu;
 760         void __iomem *cb_base, *gr1_base;
 761
 762         gr1_base = ARM_SMMU_GR1(smmu);
 763         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 764         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 765
 766         if (smmu->version > ARM_SMMU_V1) {
 767                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 768                         reg = CBA2R_RW64_64BIT;
 769                 else
 770                         reg = CBA2R_RW64_32BIT;
 771                 /* 16-bit VMIDs live in CBA2R */
 772                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 773                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 774
 775                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 776         }
 777
 778         /* CBAR */
 779         reg = cfg->cbar;
 780         if (smmu->version < ARM_SMMU_V2)
 781                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 782
 783         /*
 784          * Use the weakest shareability/memory types, so they are
 785          * overridden by the ttbcr/pte.
 786          */
 787         if (stage1) {
 788                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 789                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 790         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 791                 /* 8-bit VMIDs live in CBAR */
 792                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 793         }
 794         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 795
 796         /* TTBRs */
 797         if (stage1) {
 798                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 799
 800                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 801                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 802
 803                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 804                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
 805                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 806         } else {
 807                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 808                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 809         }
 810
 811         /* TTBCR */
 812         if (stage1) {
 813                 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 814                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 815                 if (smmu->version > ARM_SMMU_V1) {
 816                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 817                         reg |= TTBCR2_SEP_UPSTREAM;
 818                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
 819                 }
 820         } else {
 821                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 822                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 823         }
 824
 825         /* MAIRs (stage-1 only) */
 826         if (stage1) {
 827                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 828                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 829                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 830                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
 831         }
 832
 833         /* SCTLR */
 834         reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
 835         if (stage1)
 836                 reg |= SCTLR_S1_ASIDPNE;
 837 #ifdef __BIG_ENDIAN
 838         reg |= SCTLR_E;
 839 #endif
 840         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 841 }
 842
 843 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 844                                         struct arm_smmu_device *smmu)
 845 {
 846         int irq, start, ret = 0;
 847         unsigned long ias, oas;
 848         struct io_pgtable_ops *pgtbl_ops;
 849         struct io_pgtable_cfg pgtbl_cfg;
 850         enum io_pgtable_fmt fmt;
 851         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 852         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 853
 854         mutex_lock(&smmu_domain->init_mutex);
 855         if (smmu_domain->smmu)
 856                 goto out_unlock;
 857
 858         /* We're bypassing these SIDs, so don't allocate an actual context */
 859         if (domain->type == IOMMU_DOMAIN_DMA) {
 860                 smmu_domain->smmu = smmu;
 861                 goto out_unlock;
 862         }
 863
 864         /*
 865          * Mapping the requested stage onto what we support is surprisingly
 866          * complicated, mainly because the spec allows S1+S2 SMMUs without
 867          * support for nested translation. That means we end up with the
 868          * following table:
 869          *
 870          * Requested        Supported        Actual
 871          *     S1               N              S1
 872          *     S1             S1+S2            S1
 873          *     S1               S2             S2
 874          *     S1               S1             S1
 875          *     N                N              N
 876          *     N              S1+S2            S2
 877          *     N                S2             S2
 878          *     N                S1             S1
 879          *
 880          * Note that you can't actually request stage-2 mappings.
 881          */
 882         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 883                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 884         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 885                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 886
 887         /*
 888          * Choosing a suitable context format is even more fiddly. Until we
 889          * grow some way for the caller to express a preference, and/or move
 890          * the decision into the io-pgtable code where it arguably belongs,
 891          * just aim for the closest thing to the rest of the system, and hope
 892          * that the hardware isn't esoteric enough that we can't assume AArch64
 893          * support to be a superset of AArch32 support...
 894          */
 895         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 896                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 897         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 898             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 899                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 900                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 901                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 902
 903         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 904                 ret = -EINVAL;
 905                 goto out_unlock;
 906         }
 907
 908         switch (smmu_domain->stage) {
 909         case ARM_SMMU_DOMAIN_S1:
 910                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 911                 start = smmu->num_s2_context_banks;
 912                 ias = smmu->va_size;
 913                 oas = smmu->ipa_size;
 914                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 915                         fmt = ARM_64_LPAE_S1;
 916                 } else {
 917                         fmt = ARM_32_LPAE_S1;
 918                         ias = min(ias, 32UL);
 919                         oas = min(oas, 40UL);
 920                 }
 921                 break;
 922         case ARM_SMMU_DOMAIN_NESTED:
 923                 /*
 924                  * We will likely want to change this if/when KVM gets
 925                  * involved.
 926                  */
 927         case ARM_SMMU_DOMAIN_S2:
 928                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 929                 start = 0;
 930                 ias = smmu->ipa_size;
 931                 oas = smmu->pa_size;
 932                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 933                         fmt = ARM_64_LPAE_S2;
 934                 } else {
 935                         fmt = ARM_32_LPAE_S2;
 936                         ias = min(ias, 40UL);
 937                         oas = min(oas, 40UL);
 938                 }
 939                 break;
 940         default:
 941                 ret = -EINVAL;
 942                 goto out_unlock;
 943         }
 944
 945         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 946                                       smmu->num_context_banks);
 947         if (IS_ERR_VALUE(ret))
 948                 goto out_unlock;
 949
 950         cfg->cbndx = ret;
 951         if (smmu->version < ARM_SMMU_V2) {
 952                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 953                 cfg->irptndx %= smmu->num_context_irqs;
 954         } else {
 955                 cfg->irptndx = cfg->cbndx;
 956         }
 957
 958         pgtbl_cfg = (struct io_pgtable_cfg) {
 959                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 960                 .ias            = ias,
 961                 .oas            = oas,
 962                 .tlb            = &arm_smmu_gather_ops,
 963                 .iommu_dev      = smmu->dev,
 964         };
 965
 966         smmu_domain->smmu = smmu;
 967         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 968         if (!pgtbl_ops) {
 969                 ret = -ENOMEM;
 970                 goto out_clear_smmu;
 971         }
 972
 973         /* Update the domain's page sizes to reflect the page table format */
 974         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 975
 976         /* Initialise the context bank with our page table cfg */
 977         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 978
 979         /*
 980          * Request context fault interrupt. Do this last to avoid the
 981          * handler seeing a half-initialised domain state.
 982          */
 983         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 984         ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
 985                           "arm-smmu-context-fault", domain);
 986         if (IS_ERR_VALUE(ret)) {
 987                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 988                         cfg->irptndx, irq);
 989                 cfg->irptndx = INVALID_IRPTNDX;
 990         }
 991
 992         mutex_unlock(&smmu_domain->init_mutex);
 993
 994         /* Publish page table ops for map/unmap */
 995         smmu_domain->pgtbl_ops = pgtbl_ops;
 996         return 0;
 997
 998 out_clear_smmu:
 999         smmu_domain->smmu = NULL;
1000 out_unlock:
1001         mutex_unlock(&smmu_domain->init_mutex);
1002         return ret;
1003 }
1004
1005 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1006 {
1007         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1008         struct arm_smmu_device *smmu = smmu_domain->smmu;
1009         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1010         void __iomem *cb_base;
1011         int irq;
1012
1013         if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
1014                 return;
1015
1016         /*
1017          * Disable the context bank and free the page tables before freeing
1018          * it.
1019          */
1020         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1021         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1022
1023         if (cfg->irptndx != INVALID_IRPTNDX) {
1024                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1025                 free_irq(irq, domain);
1026         }
1027
1028         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1029         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1030 }
1031
1032 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1033 {
1034         struct arm_smmu_domain *smmu_domain;
1035
1036         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1037                 return NULL;
1038         /*
1039          * Allocate the domain and initialise some of its data structures.
1040          * We can't really do anything meaningful until we've added a
1041          * master.
1042          */
1043         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1044         if (!smmu_domain)
1045                 return NULL;
1046
1047         if (type == IOMMU_DOMAIN_DMA &&
1048             iommu_get_dma_cookie(&smmu_domain->domain)) {
1049                 kfree(smmu_domain);
1050                 return NULL;
1051         }
1052
1053         mutex_init(&smmu_domain->init_mutex);
1054         spin_lock_init(&smmu_domain->pgtbl_lock);
1055
1056         return &smmu_domain->domain;
1057 }
1058
1059 static void arm_smmu_domain_free(struct iommu_domain *domain)
1060 {
1061         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1062
1063         /*
1064          * Free the domain resources. We assume that all devices have
1065          * already been detached.
1066          */
1067         iommu_put_dma_cookie(domain);
1068         arm_smmu_destroy_domain_context(domain);
1069         kfree(smmu_domain);
1070 }
1071
1072 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1073                                           struct arm_smmu_master_cfg *cfg)
1074 {
1075         int i;
1076         struct arm_smmu_smr *smrs;
1077         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1078
1079         if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1080                 return 0;
1081
1082         if (cfg->smrs)
1083                 return -EEXIST;
1084
1085         smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1086         if (!smrs) {
1087                 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1088                         cfg->num_streamids);
1089                 return -ENOMEM;
1090         }
1091
1092         /* Allocate the SMRs on the SMMU */
1093         for (i = 0; i < cfg->num_streamids; ++i) {
1094                 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1095                                                   smmu->num_mapping_groups);
1096                 if (IS_ERR_VALUE(idx)) {
1097                         dev_err(smmu->dev, "failed to allocate free SMR\n");
1098                         goto err_free_smrs;
1099                 }
1100
1101                 smrs[i] = (struct arm_smmu_smr) {
1102                         .idx    = idx,
1103                         .mask   = 0, /* We don't currently share SMRs */
1104                         .id     = cfg->streamids[i],
1105                 };
1106         }
1107
1108         /* It worked! Now, poke the actual hardware */
1109         for (i = 0; i < cfg->num_streamids; ++i) {
1110                 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1111                           smrs[i].mask << SMR_MASK_SHIFT;
1112                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1113         }
1114
1115         cfg->smrs = smrs;
1116         return 0;
1117
1118 err_free_smrs:
1119         while (--i >= 0)
1120                 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1121         kfree(smrs);
1122         return -ENOSPC;
1123 }
1124
1125 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1126                                       struct arm_smmu_master_cfg *cfg)
1127 {
1128         int i;
1129         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1130         struct arm_smmu_smr *smrs = cfg->smrs;
1131
1132         if (!smrs)
1133                 return;
1134
1135         /* Invalidate the SMRs before freeing back to the allocator */
1136         for (i = 0; i < cfg->num_streamids; ++i) {
1137                 u8 idx = smrs[i].idx;
1138
1139                 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1140                 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1141         }
1142
1143         cfg->smrs = NULL;
1144         kfree(smrs);
1145 }
1146
1147 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1148                                       struct arm_smmu_master_cfg *cfg)
1149 {
1150         int i, ret;
1151         struct arm_smmu_device *smmu = smmu_domain->smmu;
1152         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1153
1154         /*
1155          * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1156          * for all devices behind the SMMU. Note that we need to take
1157          * care configuring SMRs for devices both a platform_device and
1158          * and a PCI device (i.e. a PCI host controller)
1159          */
1160         if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1161                 return 0;
1162
1163         /* Devices in an IOMMU group may already be configured */
1164         ret = arm_smmu_master_configure_smrs(smmu, cfg);
1165         if (ret)
1166                 return ret == -EEXIST ? 0 : ret;
1167
1168         for (i = 0; i < cfg->num_streamids; ++i) {
1169                 u32 idx, s2cr;
1170
1171                 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1172                 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1173                        (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1174                 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1175         }
1176
1177         return 0;
1178 }
1179
1180 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1181                                           struct arm_smmu_master_cfg *cfg)
1182 {
1183         int i;
1184         struct arm_smmu_device *smmu = smmu_domain->smmu;
1185         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1186
1187         /* An IOMMU group is torn down by the first device to be removed */
1188         if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1189                 return;
1190
1191         /*
1192          * We *must* clear the S2CR first, because freeing the SMR means
1193          * that it can be re-allocated immediately.
1194          */
1195         for (i = 0; i < cfg->num_streamids; ++i) {
1196                 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1197                 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1198
1199                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1200         }
1201
1202         arm_smmu_master_free_smrs(smmu, cfg);
1203 }
1204
1205 static void arm_smmu_detach_dev(struct device *dev,
1206                                 struct arm_smmu_master_cfg *cfg)
1207 {
1208         struct iommu_domain *domain = dev->archdata.iommu;
1209         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1210
1211         dev->archdata.iommu = NULL;
1212         arm_smmu_domain_remove_master(smmu_domain, cfg);
1213 }
1214
1215 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1216 {
1217         int ret;
1218         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1219         struct arm_smmu_device *smmu;
1220         struct arm_smmu_master_cfg *cfg;
1221
1222         smmu = find_smmu_for_device(dev);
1223         if (!smmu) {
1224                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1225                 return -ENXIO;
1226         }
1227
1228         /* Ensure that the domain is finalised */
1229         ret = arm_smmu_init_domain_context(domain, smmu);
1230         if (IS_ERR_VALUE(ret))
1231                 return ret;
1232
1233         /*
1234          * Sanity check the domain. We don't support domains across
1235          * different SMMUs.
1236          */
1237         if (smmu_domain->smmu != smmu) {
1238                 dev_err(dev,
1239                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1240                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1241                 return -EINVAL;
1242         }
1243
1244         /* Looks ok, so add the device to the domain */
1245         cfg = find_smmu_master_cfg(dev);
1246         if (!cfg)
1247                 return -ENODEV;
1248
1249         /* Detach the dev from its current domain */
1250         if (dev->archdata.iommu)
1251                 arm_smmu_detach_dev(dev, cfg);
1252
1253         ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1254         if (!ret)
1255                 dev->archdata.iommu = domain;
1256         return ret;
1257 }
1258
1259 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1260                         phys_addr_t paddr, size_t size, int prot)
1261 {
1262         int ret;
1263         unsigned long flags;
1264         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1265         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1266
1267         if (!ops)
1268                 return -ENODEV;
1269
1270         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1271         ret = ops->map(ops, iova, paddr, size, prot);
1272         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1273         return ret;
1274 }
1275
1276 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1277                              size_t size)
1278 {
1279         size_t ret;
1280         unsigned long flags;
1281         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1282         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1283
1284         if (!ops)
1285                 return 0;
1286
1287         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1288         ret = ops->unmap(ops, iova, size);
1289         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1290         return ret;
1291 }
1292
1293 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1294                                               dma_addr_t iova)
1295 {
1296         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1297         struct arm_smmu_device *smmu = smmu_domain->smmu;
1298         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1299         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1300         struct device *dev = smmu->dev;
1301         void __iomem *cb_base;
1302         u32 tmp;
1303         u64 phys;
1304         unsigned long va;
1305
1306         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1307
1308         /* ATS1 registers can only be written atomically */
1309         va = iova & ~0xfffUL;
1310         if (smmu->version == ARM_SMMU_V2)
1311                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1312         else /* Register is only 32-bit in v1 */
1313                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1314
1315         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1316                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1317                 dev_err(dev,
1318                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1319                         &iova);
1320                 return ops->iova_to_phys(ops, iova);
1321         }
1322
1323         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1324         if (phys & CB_PAR_F) {
1325                 dev_err(dev, "translation fault!\n");
1326                 dev_err(dev, "PAR = 0x%llx\n", phys);
1327                 return 0;
1328         }
1329
1330         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1331 }
1332
1333 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1334                                         dma_addr_t iova)
1335 {
1336         phys_addr_t ret;
1337         unsigned long flags;
1338         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1339         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1340
1341         if (!ops)
1342                 return 0;
1343
1344         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1345         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1346                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1347                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1348         } else {
1349                 ret = ops->iova_to_phys(ops, iova);
1350         }
1351
1352         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1353
1354         return ret;
1355 }
1356
1357 static bool arm_smmu_capable(enum iommu_cap cap)
1358 {
1359         switch (cap) {
1360         case IOMMU_CAP_CACHE_COHERENCY:
1361                 /*
1362                  * Return true here as the SMMU can always send out coherent
1363                  * requests.
1364                  */
1365                 return true;
1366         case IOMMU_CAP_INTR_REMAP:
1367                 return true; /* MSIs are just memory writes */
1368         case IOMMU_CAP_NOEXEC:
1369                 return true;
1370         default:
1371                 return false;
1372         }
1373 }
1374
1375 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1376 {
1377         *((u16 *)data) = alias;
1378         return 0; /* Continue walking */
1379 }
1380
1381 static void __arm_smmu_release_pci_iommudata(void *data)
1382 {
1383         kfree(data);
1384 }
1385
1386 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1387                                     struct iommu_group *group)
1388 {
1389         struct arm_smmu_master_cfg *cfg;
1390         u16 sid;
1391         int i;
1392
1393         cfg = iommu_group_get_iommudata(group);
1394         if (!cfg) {
1395                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1396                 if (!cfg)
1397                         return -ENOMEM;
1398
1399                 iommu_group_set_iommudata(group, cfg,
1400                                           __arm_smmu_release_pci_iommudata);
1401         }
1402
1403         if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1404                 return -ENOSPC;
1405
1406         /*
1407          * Assume Stream ID == Requester ID for now.
1408          * We need a way to describe the ID mappings in FDT.
1409          */
1410         pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1411         for (i = 0; i < cfg->num_streamids; ++i)
1412                 if (cfg->streamids[i] == sid)
1413                         break;
1414
1415         /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1416         if (i == cfg->num_streamids)
1417                 cfg->streamids[cfg->num_streamids++] = sid;
1418
1419         return 0;
1420 }
1421
1422 static int arm_smmu_init_platform_device(struct device *dev,
1423                                          struct iommu_group *group)
1424 {
1425         struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1426         struct arm_smmu_master *master;
1427
1428         if (!smmu)
1429                 return -ENODEV;
1430
1431         master = find_smmu_master(smmu, dev->of_node);
1432         if (!master)
1433                 return -ENODEV;
1434
1435         iommu_group_set_iommudata(group, &master->cfg, NULL);
1436
1437         return 0;
1438 }
1439
1440 static int arm_smmu_add_device(struct device *dev)
1441 {
1442         struct iommu_group *group;
1443
1444         group = iommu_group_get_for_dev(dev);
1445         if (IS_ERR(group))
1446                 return PTR_ERR(group);
1447
1448         iommu_group_put(group);
1449         return 0;
1450 }
1451
1452 static void arm_smmu_remove_device(struct device *dev)
1453 {
1454         iommu_group_remove_device(dev);
1455 }
1456
1457 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1458 {
1459         struct iommu_group *group;
1460         int ret;
1461
1462         if (dev_is_pci(dev))
1463                 group = pci_device_group(dev);
1464         else
1465                 group = generic_device_group(dev);
1466
1467         if (IS_ERR(group))
1468                 return group;
1469
1470         if (dev_is_pci(dev))
1471                 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1472         else
1473                 ret = arm_smmu_init_platform_device(dev, group);
1474
1475         if (ret) {
1476                 iommu_group_put(group);
1477                 group = ERR_PTR(ret);
1478         }
1479
1480         return group;
1481 }
1482
1483 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1484                                     enum iommu_attr attr, void *data)
1485 {
1486         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1487
1488         switch (attr) {
1489         case DOMAIN_ATTR_NESTING:
1490                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1491                 return 0;
1492         default:
1493                 return -ENODEV;
1494         }
1495 }
1496
1497 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1498                                     enum iommu_attr attr, void *data)
1499 {
1500         int ret = 0;
1501         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1502
1503         mutex_lock(&smmu_domain->init_mutex);
1504
1505         switch (attr) {
1506         case DOMAIN_ATTR_NESTING:
1507                 if (smmu_domain->smmu) {
1508                         ret = -EPERM;
1509                         goto out_unlock;
1510                 }
1511
1512                 if (*(int *)data)
1513                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1514                 else
1515                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1516
1517                 break;
1518         default:
1519                 ret = -ENODEV;
1520         }
1521
1522 out_unlock:
1523         mutex_unlock(&smmu_domain->init_mutex);
1524         return ret;
1525 }
1526
1527 static struct iommu_ops arm_smmu_ops = {
1528         .capable                = arm_smmu_capable,
1529         .domain_alloc           = arm_smmu_domain_alloc,
1530         .domain_free            = arm_smmu_domain_free,
1531         .attach_dev             = arm_smmu_attach_dev,
1532         .map                    = arm_smmu_map,
1533         .unmap                  = arm_smmu_unmap,
1534         .map_sg                 = default_iommu_map_sg,
1535         .iova_to_phys           = arm_smmu_iova_to_phys,
1536         .add_device             = arm_smmu_add_device,
1537         .remove_device          = arm_smmu_remove_device,
1538         .device_group           = arm_smmu_device_group,
1539         .domain_get_attr        = arm_smmu_domain_get_attr,
1540         .domain_set_attr        = arm_smmu_domain_set_attr,
1541         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1542 };
1543
1544 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1545 {
1546         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1547         void __iomem *cb_base;
1548         int i = 0;
1549         u32 reg, major;
1550
1551         /* clear global FSR */
1552         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1553         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1554
1555         /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1556         reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1557         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1558                 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1559                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1560         }
1561
1562         /*
1563          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1564          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1565          * bit is only present in MMU-500r2 onwards.
1566          */
1567         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1568         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1569         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1570                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1571                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1572                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1573         }
1574
1575         /* Make sure all context banks are disabled and clear CB_FSR  */
1576         for (i = 0; i < smmu->num_context_banks; ++i) {
1577                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1578                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1579                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1580                 /*
1581                  * Disable MMU-500's not-particularly-beneficial next-page
1582                  * prefetcher for the sake of errata #841119 and #826419.
1583                  */
1584                 if (smmu->model == ARM_MMU500) {
1585                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1586                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1587                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1588                 }
1589         }
1590
1591         /* Invalidate the TLB, just in case */
1592         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1593         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1594
1595         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1596
1597         /* Enable fault reporting */
1598         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1599
1600         /* Disable TLB broadcasting. */
1601         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1602
1603         /* Enable client access, handling unmatched streams as appropriate */
1604         reg &= ~sCR0_CLIENTPD;
1605         if (disable_bypass)
1606                 reg |= sCR0_USFCFG;
1607         else
1608                 reg &= ~sCR0_USFCFG;
1609
1610         /* Disable forced broadcasting */
1611         reg &= ~sCR0_FB;
1612
1613         /* Don't upgrade barriers */
1614         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1615
1616         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1617                 reg |= sCR0_VMID16EN;
1618
1619         /* Push the button */
1620         __arm_smmu_tlb_sync(smmu);
1621         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1622 }
1623
1624 static int arm_smmu_id_size_to_bits(int size)
1625 {
1626         switch (size) {
1627         case 0:
1628                 return 32;
1629         case 1:
1630                 return 36;
1631         case 2:
1632                 return 40;
1633         case 3:
1634                 return 42;
1635         case 4:
1636                 return 44;
1637         case 5:
1638         default:
1639                 return 48;
1640         }
1641 }
1642
1643 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1644 {
1645         unsigned long size;
1646         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1647         u32 id;
1648         bool cttw_dt, cttw_reg;
1649
1650         dev_notice(smmu->dev, "probing hardware configuration...\n");
1651         dev_notice(smmu->dev, "SMMUv%d with:\n",
1652                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1653
1654         /* ID0 */
1655         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1656
1657         /* Restrict available stages based on module parameter */
1658         if (force_stage == 1)
1659                 id &= ~(ID0_S2TS | ID0_NTS);
1660         else if (force_stage == 2)
1661                 id &= ~(ID0_S1TS | ID0_NTS);
1662
1663         if (id & ID0_S1TS) {
1664                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1665                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1666         }
1667
1668         if (id & ID0_S2TS) {
1669                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1670                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1671         }
1672
1673         if (id & ID0_NTS) {
1674                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1675                 dev_notice(smmu->dev, "\tnested translation\n");
1676         }
1677
1678         if (!(smmu->features &
1679                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1680                 dev_err(smmu->dev, "\tno translation support!\n");
1681                 return -ENODEV;
1682         }
1683
1684         if ((id & ID0_S1TS) &&
1685                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1686                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1687                 dev_notice(smmu->dev, "\taddress translation ops\n");
1688         }
1689
1690         /*
1691          * In order for DMA API calls to work properly, we must defer to what
1692          * the DT says about coherency, regardless of what the hardware claims.
1693          * Fortunately, this also opens up a workaround for systems where the
1694          * ID register value has ended up configured incorrectly.
1695          */
1696         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1697         cttw_reg = !!(id & ID0_CTTW);
1698         if (cttw_dt)
1699                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1700         if (cttw_dt || cttw_reg)
1701                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1702                            cttw_dt ? "" : "non-");
1703         if (cttw_dt != cttw_reg)
1704                 dev_notice(smmu->dev,
1705                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1706
1707         if (id & ID0_SMS) {
1708                 u32 smr, sid, mask;
1709
1710                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1711                 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1712                                            ID0_NUMSMRG_MASK;
1713                 if (smmu->num_mapping_groups == 0) {
1714                         dev_err(smmu->dev,
1715                                 "stream-matching supported, but no SMRs present!\n");
1716                         return -ENODEV;
1717                 }
1718
1719                 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1720                 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1721                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1722                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1723
1724                 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1725                 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1726                 if ((mask & sid) != sid) {
1727                         dev_err(smmu->dev,
1728                                 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1729                                 mask, sid);
1730                         return -ENODEV;
1731                 }
1732
1733                 dev_notice(smmu->dev,
1734                            "\tstream matching with %u register groups, mask 0x%x",
1735                            smmu->num_mapping_groups, mask);
1736         } else {
1737                 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1738                                            ID0_NUMSIDB_MASK;
1739         }
1740
1741         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1742                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1743                 if (!(id & ID0_PTFS_NO_AARCH32S))
1744                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1745         }
1746
1747         /* ID1 */
1748         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1749         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1750
1751         /* Check for size mismatch of SMMU address space from mapped region */
1752         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1753         size *= 2 << smmu->pgshift;
1754         if (smmu->size != size)
1755                 dev_warn(smmu->dev,
1756                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1757                         size, smmu->size);
1758
1759         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1760         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1761         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1762                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1763                 return -ENODEV;
1764         }
1765         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1766                    smmu->num_context_banks, smmu->num_s2_context_banks);
1767         /*
1768          * Cavium CN88xx erratum #27704.
1769          * Ensure ASID and VMID allocation is unique across all SMMUs in
1770          * the system.
1771          */
1772         if (smmu->model == CAVIUM_SMMUV2) {
1773                 smmu->cavium_id_base =
1774                         atomic_add_return(smmu->num_context_banks,
1775                                           &cavium_smmu_context_count);
1776                 smmu->cavium_id_base -= smmu->num_context_banks;
1777         }
1778
1779         /* ID2 */
1780         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1781         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1782         smmu->ipa_size = size;
1783
1784         /* The output mask is also applied for bypass */
1785         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1786         smmu->pa_size = size;
1787
1788         if (id & ID2_VMID16)
1789                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1790
1791         /*
1792          * What the page table walker can address actually depends on which
1793          * descriptor format is in use, but since a) we don't know that yet,
1794          * and b) it can vary per context bank, this will have to do...
1795          */
1796         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1797                 dev_warn(smmu->dev,
1798                          "failed to set DMA mask for table walker\n");
1799
1800         if (smmu->version < ARM_SMMU_V2) {
1801                 smmu->va_size = smmu->ipa_size;
1802                 if (smmu->version == ARM_SMMU_V1_64K)
1803                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1804         } else {
1805                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1806                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1807                 if (id & ID2_PTFS_4K)
1808                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1809                 if (id & ID2_PTFS_16K)
1810                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1811                 if (id & ID2_PTFS_64K)
1812                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1813         }
1814
1815         /* Now we've corralled the various formats, what'll it do? */
1816         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1817                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1818         if (smmu->features &
1819             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1820                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1821         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1822                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1823         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1824                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1825
1826         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1827                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1828         else
1829                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1830         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1831                    smmu->pgsize_bitmap);
1832
1833
1834         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1835                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1836                            smmu->va_size, smmu->ipa_size);
1837
1838         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1839                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1840                            smmu->ipa_size, smmu->pa_size);
1841
1842         return 0;
1843 }
1844
1845 struct arm_smmu_match_data {
1846         enum arm_smmu_arch_version version;
1847         enum arm_smmu_implementation model;
1848 };
1849
1850 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1851 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1852
1853 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1854 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1855 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1856 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1857 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1858
1859 static const struct of_device_id arm_smmu_of_match[] = {
1860         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1861         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1862         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1863         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1864         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1865         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1866         { },
1867 };
1868 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1869
1870 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1871 {
1872         const struct of_device_id *of_id;
1873         const struct arm_smmu_match_data *data;
1874         struct resource *res;
1875         struct arm_smmu_device *smmu;
1876         struct device *dev = &pdev->dev;
1877         struct rb_node *node;
1878         struct of_phandle_args masterspec;
1879         int num_irqs, i, err;
1880
1881         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1882         if (!smmu) {
1883                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1884                 return -ENOMEM;
1885         }
1886         smmu->dev = dev;
1887
1888         of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1889         data = of_id->data;
1890         smmu->version = data->version;
1891         smmu->model = data->model;
1892
1893         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1894         smmu->base = devm_ioremap_resource(dev, res);
1895         if (IS_ERR(smmu->base))
1896                 return PTR_ERR(smmu->base);
1897         smmu->size = resource_size(res);
1898
1899         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1900                                  &smmu->num_global_irqs)) {
1901                 dev_err(dev, "missing #global-interrupts property\n");
1902                 return -ENODEV;
1903         }
1904
1905         num_irqs = 0;
1906         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1907                 num_irqs++;
1908                 if (num_irqs > smmu->num_global_irqs)
1909                         smmu->num_context_irqs++;
1910         }
1911
1912         if (!smmu->num_context_irqs) {
1913                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1914                         num_irqs, smmu->num_global_irqs + 1);
1915                 return -ENODEV;
1916         }
1917
1918         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1919                                   GFP_KERNEL);
1920         if (!smmu->irqs) {
1921                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1922                 return -ENOMEM;
1923         }
1924
1925         for (i = 0; i < num_irqs; ++i) {
1926                 int irq = platform_get_irq(pdev, i);
1927
1928                 if (irq < 0) {
1929                         dev_err(dev, "failed to get irq index %d\n", i);
1930                         return -ENODEV;
1931                 }
1932                 smmu->irqs[i] = irq;
1933         }
1934
1935         err = arm_smmu_device_cfg_probe(smmu);
1936         if (err)
1937                 return err;
1938
1939         i = 0;
1940         smmu->masters = RB_ROOT;
1941         while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
1942                                            "#stream-id-cells", i,
1943                                            &masterspec)) {
1944                 err = register_smmu_master(smmu, dev, &masterspec);
1945                 if (err) {
1946                         dev_err(dev, "failed to add master %s\n",
1947                                 masterspec.np->name);
1948                         goto out_put_masters;
1949                 }
1950
1951                 i++;
1952         }
1953         dev_notice(dev, "registered %d master devices\n", i);
1954
1955         parse_driver_options(smmu);
1956
1957         if (smmu->version == ARM_SMMU_V2 &&
1958             smmu->num_context_banks != smmu->num_context_irqs) {
1959                 dev_err(dev,
1960                         "found only %d context interrupt(s) but %d required\n",
1961                         smmu->num_context_irqs, smmu->num_context_banks);
1962                 err = -ENODEV;
1963                 goto out_put_masters;
1964         }
1965
1966         for (i = 0; i < smmu->num_global_irqs; ++i) {
1967                 err = request_irq(smmu->irqs[i],
1968                                   arm_smmu_global_fault,
1969                                   IRQF_SHARED,
1970                                   "arm-smmu global fault",
1971                                   smmu);
1972                 if (err) {
1973                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1974                                 i, smmu->irqs[i]);
1975                         goto out_free_irqs;
1976                 }
1977         }
1978
1979         INIT_LIST_HEAD(&smmu->list);
1980         spin_lock(&arm_smmu_devices_lock);
1981         list_add(&smmu->list, &arm_smmu_devices);
1982         spin_unlock(&arm_smmu_devices_lock);
1983
1984         arm_smmu_device_reset(smmu);
1985         return 0;
1986
1987 out_free_irqs:
1988         while (i--)
1989                 free_irq(smmu->irqs[i], smmu);
1990
1991 out_put_masters:
1992         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1993                 struct arm_smmu_master *master
1994                         = container_of(node, struct arm_smmu_master, node);
1995                 of_node_put(master->of_node);
1996         }
1997
1998         return err;
1999 }
2000
2001 static int arm_smmu_device_remove(struct platform_device *pdev)
2002 {
2003         int i;
2004         struct device *dev = &pdev->dev;
2005         struct arm_smmu_device *curr, *smmu = NULL;
2006         struct rb_node *node;
2007
2008         spin_lock(&arm_smmu_devices_lock);
2009         list_for_each_entry(curr, &arm_smmu_devices, list) {
2010                 if (curr->dev == dev) {
2011                         smmu = curr;
2012                         list_del(&smmu->list);
2013                         break;
2014                 }
2015         }
2016         spin_unlock(&arm_smmu_devices_lock);
2017
2018         if (!smmu)
2019                 return -ENODEV;
2020
2021         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2022                 struct arm_smmu_master *master
2023                         = container_of(node, struct arm_smmu_master, node);
2024                 of_node_put(master->of_node);
2025         }
2026
2027         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2028                 dev_err(dev, "removing device with active domains!\n");
2029
2030         for (i = 0; i < smmu->num_global_irqs; ++i)
2031                 free_irq(smmu->irqs[i], smmu);
2032
2033         /* Turn the thing off */
2034         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2035         return 0;
2036 }
2037
2038 static struct platform_driver arm_smmu_driver = {
2039         .driver = {
2040                 .name           = "arm-smmu",
2041                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2042         },
2043         .probe  = arm_smmu_device_dt_probe,
2044         .remove = arm_smmu_device_remove,
2045 };
2046
2047 static int __init arm_smmu_init(void)
2048 {
2049         struct device_node *np;
2050         int ret;
2051
2052         /*
2053          * Play nice with systems that don't have an ARM SMMU by checking that
2054          * an ARM SMMU exists in the system before proceeding with the driver
2055          * and IOMMU bus operation registration.
2056          */
2057         np = of_find_matching_node(NULL, arm_smmu_of_match);
2058         if (!np)
2059                 return 0;
2060
2061         of_node_put(np);
2062
2063         ret = platform_driver_register(&arm_smmu_driver);
2064         if (ret)
2065                 return ret;
2066
2067         /* Oh, for a proper bus abstraction */
2068         if (!iommu_present(&platform_bus_type))
2069                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2070
2071 #ifdef CONFIG_ARM_AMBA
2072         if (!iommu_present(&amba_bustype))
2073                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2074 #endif
2075
2076 #ifdef CONFIG_PCI
2077         if (!iommu_present(&pci_bus_type))
2078                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2079 #endif
2080
2081         return 0;
2082 }
2083
2084 static void __exit arm_smmu_exit(void)
2085 {
2086         return platform_driver_unregister(&arm_smmu_driver);
2087 }
2088
2089 subsys_initcall(arm_smmu_init);
2090 module_exit(arm_smmu_exit);
2091
2092 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2093 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2094 MODULE_LICENSE("GPL v2");