Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/drivers/iommu/arm-smmu.c

  1 /*
  2  * IOMMU API for ARM architected SMMU implementations.
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  *
  8  * This program is distributed in the hope that it will be useful,
  9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 11  * GNU General Public License for more details.
 12  *
 13  * You should have received a copy of the GNU General Public License
 14  * along with this program; if not, write to the Free Software
 15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 16  *
 17  * Copyright (C) 2013 ARM Limited
 18  *
 19  * Author: Will Deacon <will.deacon@arm.com>
 20  *
 21  * This driver currently supports:
 22  *      - SMMUv1 and v2 implementations
 23  *      - Stream-matching and stream-indexing
 24  *      - v7/v8 long-descriptor format
 25  *      - Non-secure access to the SMMU
 26  *      - Context fault reporting
 27  */
 28 
 29 #define pr_fmt(fmt) "arm-smmu: " fmt
 30 
 31 #include <linux/acpi.h>
 32 #include <linux/acpi_iort.h>
 33 #include <linux/atomic.h>
 34 #include <linux/delay.h>
 35 #include <linux/dma-iommu.h>
 36 #include <linux/dma-mapping.h>
 37 #include <linux/err.h>
 38 #include <linux/interrupt.h>
 39 #include <linux/io.h>
 40 #include <linux/io-64-nonatomic-hi-lo.h>
 41 #include <linux/iommu.h>
 42 #include <linux/iopoll.h>
 43 #include <linux/module.h>
 44 #include <linux/of.h>
 45 #include <linux/of_address.h>
 46 #include <linux/of_device.h>
 47 #include <linux/of_iommu.h>
 48 #include <linux/pci.h>
 49 #include <linux/platform_device.h>
 50 #include <linux/slab.h>
 51 #include <linux/spinlock.h>
 52 
 53 #include <linux/amba/bus.h>
 54 
 55 #include "io-pgtable.h"
 56 
 57 /* Maximum number of context banks per SMMU */
 58 #define ARM_SMMU_MAX_CBS                128
 59 
 60 /* SMMU global address space */
 61 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
 62 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
 63 
 64 /*
 65  * SMMU global address space with conditional offset to access secure
 66  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
 67  * nsGFSYNR0: 0x450)
 68  */
 69 #define ARM_SMMU_GR0_NS(smmu)                                           \
 70         ((smmu)->base +                                                 \
 71                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
 72                         ? 0x400 : 0))
 73 
 74 /*
 75  * Some 64-bit registers only make sense to write atomically, but in such
 76  * cases all the data relevant to AArch32 formats lies within the lower word,
 77  * therefore this actually makes more sense than it might first appear.
 78  */
 79 #ifdef CONFIG_64BIT
 80 #define smmu_write_atomic_lq            writeq_relaxed
 81 #else
 82 #define smmu_write_atomic_lq            writel_relaxed
 83 #endif
 84 
 85 /* Configuration registers */
 86 #define ARM_SMMU_GR0_sCR0               0x0
 87 #define sCR0_CLIENTPD                   (1 << 0)
 88 #define sCR0_GFRE                       (1 << 1)
 89 #define sCR0_GFIE                       (1 << 2)
 90 #define sCR0_GCFGFRE                    (1 << 4)
 91 #define sCR0_GCFGFIE                    (1 << 5)
 92 #define sCR0_USFCFG                     (1 << 10)
 93 #define sCR0_VMIDPNE                    (1 << 11)
 94 #define sCR0_PTM                        (1 << 12)
 95 #define sCR0_FB                         (1 << 13)
 96 #define sCR0_VMID16EN                   (1 << 31)
 97 #define sCR0_BSU_SHIFT                  14
 98 #define sCR0_BSU_MASK                   0x3
 99 
100 /* Auxiliary Configuration register */
101 #define ARM_SMMU_GR0_sACR               0x10
102 
103 /* Identification registers */
104 #define ARM_SMMU_GR0_ID0                0x20
105 #define ARM_SMMU_GR0_ID1                0x24
106 #define ARM_SMMU_GR0_ID2                0x28
107 #define ARM_SMMU_GR0_ID3                0x2c
108 #define ARM_SMMU_GR0_ID4                0x30
109 #define ARM_SMMU_GR0_ID5                0x34
110 #define ARM_SMMU_GR0_ID6                0x38
111 #define ARM_SMMU_GR0_ID7                0x3c
112 #define ARM_SMMU_GR0_sGFSR              0x48
113 #define ARM_SMMU_GR0_sGFSYNR0           0x50
114 #define ARM_SMMU_GR0_sGFSYNR1           0x54
115 #define ARM_SMMU_GR0_sGFSYNR2           0x58
116 
117 #define ID0_S1TS                        (1 << 30)
118 #define ID0_S2TS                        (1 << 29)
119 #define ID0_NTS                         (1 << 28)
120 #define ID0_SMS                         (1 << 27)
121 #define ID0_ATOSNS                      (1 << 26)
122 #define ID0_PTFS_NO_AARCH32             (1 << 25)
123 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
124 #define ID0_CTTW                        (1 << 14)
125 #define ID0_NUMIRPT_SHIFT               16
126 #define ID0_NUMIRPT_MASK                0xff
127 #define ID0_NUMSIDB_SHIFT               9
128 #define ID0_NUMSIDB_MASK                0xf
129 #define ID0_NUMSMRG_SHIFT               0
130 #define ID0_NUMSMRG_MASK                0xff
131 
132 #define ID1_PAGESIZE                    (1 << 31)
133 #define ID1_NUMPAGENDXB_SHIFT           28
134 #define ID1_NUMPAGENDXB_MASK            7
135 #define ID1_NUMS2CB_SHIFT               16
136 #define ID1_NUMS2CB_MASK                0xff
137 #define ID1_NUMCB_SHIFT                 0
138 #define ID1_NUMCB_MASK                  0xff
139 
140 #define ID2_OAS_SHIFT                   4
141 #define ID2_OAS_MASK                    0xf
142 #define ID2_IAS_SHIFT                   0
143 #define ID2_IAS_MASK                    0xf
144 #define ID2_UBS_SHIFT                   8
145 #define ID2_UBS_MASK                    0xf
146 #define ID2_PTFS_4K                     (1 << 12)
147 #define ID2_PTFS_16K                    (1 << 13)
148 #define ID2_PTFS_64K                    (1 << 14)
149 #define ID2_VMID16                      (1 << 15)
150 
151 #define ID7_MAJOR_SHIFT                 4
152 #define ID7_MAJOR_MASK                  0xf
153 
154 /* Global TLB invalidation */
155 #define ARM_SMMU_GR0_TLBIVMID           0x64
156 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
157 #define ARM_SMMU_GR0_TLBIALLH           0x6c
158 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
159 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
160 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
161 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
162 
163 /* Stream mapping registers */
164 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
165 #define SMR_VALID                       (1 << 31)
166 #define SMR_MASK_SHIFT                  16
167 #define SMR_ID_SHIFT                    0
168 
169 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
170 #define S2CR_CBNDX_SHIFT                0
171 #define S2CR_CBNDX_MASK                 0xff
172 #define S2CR_TYPE_SHIFT                 16
173 #define S2CR_TYPE_MASK                  0x3
174 enum arm_smmu_s2cr_type {
175         S2CR_TYPE_TRANS,
176         S2CR_TYPE_BYPASS,
177         S2CR_TYPE_FAULT,
178 };
179 
180 #define S2CR_PRIVCFG_SHIFT              24
181 #define S2CR_PRIVCFG_MASK               0x3
182 enum arm_smmu_s2cr_privcfg {
183         S2CR_PRIVCFG_DEFAULT,
184         S2CR_PRIVCFG_DIPAN,
185         S2CR_PRIVCFG_UNPRIV,
186         S2CR_PRIVCFG_PRIV,
187 };
188 
189 /* Context bank attribute registers */
190 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
191 #define CBAR_VMID_SHIFT                 0
192 #define CBAR_VMID_MASK                  0xff
193 #define CBAR_S1_BPSHCFG_SHIFT           8
194 #define CBAR_S1_BPSHCFG_MASK            3
195 #define CBAR_S1_BPSHCFG_NSH             3
196 #define CBAR_S1_MEMATTR_SHIFT           12
197 #define CBAR_S1_MEMATTR_MASK            0xf
198 #define CBAR_S1_MEMATTR_WB              0xf
199 #define CBAR_TYPE_SHIFT                 16
200 #define CBAR_TYPE_MASK                  0x3
201 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
202 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
203 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
204 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
205 #define CBAR_IRPTNDX_SHIFT              24
206 #define CBAR_IRPTNDX_MASK               0xff
207 
208 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
209 #define CBA2R_RW64_32BIT                (0 << 0)
210 #define CBA2R_RW64_64BIT                (1 << 0)
211 #define CBA2R_VMID_SHIFT                16
212 #define CBA2R_VMID_MASK                 0xffff
213 
214 /* Translation context bank */
215 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
216 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
217 
218 #define ARM_SMMU_CB_SCTLR               0x0
219 #define ARM_SMMU_CB_ACTLR               0x4
220 #define ARM_SMMU_CB_RESUME              0x8
221 #define ARM_SMMU_CB_TTBCR2              0x10
222 #define ARM_SMMU_CB_TTBR0               0x20
223 #define ARM_SMMU_CB_TTBR1               0x28
224 #define ARM_SMMU_CB_TTBCR               0x30
225 #define ARM_SMMU_CB_CONTEXTIDR          0x34
226 #define ARM_SMMU_CB_S1_MAIR0            0x38
227 #define ARM_SMMU_CB_S1_MAIR1            0x3c
228 #define ARM_SMMU_CB_PAR                 0x50
229 #define ARM_SMMU_CB_FSR                 0x58
230 #define ARM_SMMU_CB_FAR                 0x60
231 #define ARM_SMMU_CB_FSYNR0              0x68
232 #define ARM_SMMU_CB_S1_TLBIVA           0x600
233 #define ARM_SMMU_CB_S1_TLBIASID         0x610
234 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
235 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
236 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
237 #define ARM_SMMU_CB_ATS1PR              0x800
238 #define ARM_SMMU_CB_ATSR                0x8f0
239 
240 #define SCTLR_S1_ASIDPNE                (1 << 12)
241 #define SCTLR_CFCFG                     (1 << 7)
242 #define SCTLR_CFIE                      (1 << 6)
243 #define SCTLR_CFRE                      (1 << 5)
244 #define SCTLR_E                         (1 << 4)
245 #define SCTLR_AFE                       (1 << 2)
246 #define SCTLR_TRE                       (1 << 1)
247 #define SCTLR_M                         (1 << 0)
248 
249 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
250 
251 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
252 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
253 
254 #define CB_PAR_F                        (1 << 0)
255 
256 #define ATSR_ACTIVE                     (1 << 0)
257 
258 #define RESUME_RETRY                    (0 << 0)
259 #define RESUME_TERMINATE                (1 << 0)
260 
261 #define TTBCR2_SEP_SHIFT                15
262 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
263 
264 #define TTBRn_ASID_SHIFT                48
265 
266 #define FSR_MULTI                       (1 << 31)
267 #define FSR_SS                          (1 << 30)
268 #define FSR_UUT                         (1 << 8)
269 #define FSR_ASF                         (1 << 7)
270 #define FSR_TLBLKF                      (1 << 6)
271 #define FSR_TLBMCF                      (1 << 5)
272 #define FSR_EF                          (1 << 4)
273 #define FSR_PF                          (1 << 3)
274 #define FSR_AFF                         (1 << 2)
275 #define FSR_TF                          (1 << 1)
276 
277 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
278                                          FSR_TLBMCF | FSR_TLBLKF)
279 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
280                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
281 
282 #define FSYNR0_WNR                      (1 << 4)
283 
284 static int force_stage;
285 module_param(force_stage, int, S_IRUGO);
286 MODULE_PARM_DESC(force_stage,
287         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
288 static bool disable_bypass;
289 module_param(disable_bypass, bool, S_IRUGO);
290 MODULE_PARM_DESC(disable_bypass,
291         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
292 
293 enum arm_smmu_arch_version {
294         ARM_SMMU_V1,
295         ARM_SMMU_V1_64K,
296         ARM_SMMU_V2,
297 };
298 
299 enum arm_smmu_implementation {
300         GENERIC_SMMU,
301         ARM_MMU500,
302         CAVIUM_SMMUV2,
303 };
304 
305 struct arm_smmu_s2cr {
306         struct iommu_group              *group;
307         int                             count;
308         enum arm_smmu_s2cr_type         type;
309         enum arm_smmu_s2cr_privcfg      privcfg;
310         u8                              cbndx;
311 };
312 
313 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
314         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
315 }
316 
317 struct arm_smmu_smr {
318         u16                             mask;
319         u16                             id;
320         bool                            valid;
321 };
322 
323 struct arm_smmu_master_cfg {
324         struct arm_smmu_device          *smmu;
325         s16                             smendx[];
326 };
327 #define INVALID_SMENDX                  -1
328 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
329 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
330 #define fwspec_smendx(fw, i) \
331         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
332 #define for_each_cfg_sme(fw, i, idx) \
333         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
334 
335 struct arm_smmu_device {
336         struct device                   *dev;
337 
338         void __iomem                    *base;
339         unsigned long                   size;
340         unsigned long                   pgshift;
341 
342 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
343 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
344 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
345 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
346 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
347 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
348 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
349 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
350 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
351 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
352 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
353 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
354         u32                             features;
355 
356 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
357         u32                             options;
358         enum arm_smmu_arch_version      version;
359         enum arm_smmu_implementation    model;
360 
361         u32                             num_context_banks;
362         u32                             num_s2_context_banks;
363         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
364         atomic_t                        irptndx;
365 
366         u32                             num_mapping_groups;
367         u16                             streamid_mask;
368         u16                             smr_mask_mask;
369         struct arm_smmu_smr             *smrs;
370         struct arm_smmu_s2cr            *s2crs;
371         struct mutex                    stream_map_mutex;
372 
373         unsigned long                   va_size;
374         unsigned long                   ipa_size;
375         unsigned long                   pa_size;
376         unsigned long                   pgsize_bitmap;
377 
378         u32                             num_global_irqs;
379         u32                             num_context_irqs;
380         unsigned int                    *irqs;
381 
382         u32                             cavium_id_base; /* Specific to Cavium */
383 };
384 
385 enum arm_smmu_context_fmt {
386         ARM_SMMU_CTX_FMT_NONE,
387         ARM_SMMU_CTX_FMT_AARCH64,
388         ARM_SMMU_CTX_FMT_AARCH32_L,
389         ARM_SMMU_CTX_FMT_AARCH32_S,
390 };
391 
392 struct arm_smmu_cfg {
393         u8                              cbndx;
394         u8                              irptndx;
395         u32                             cbar;
396         enum arm_smmu_context_fmt       fmt;
397 };
398 #define INVALID_IRPTNDX                 0xff
399 
400 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
401 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
402 
403 enum arm_smmu_domain_stage {
404         ARM_SMMU_DOMAIN_S1 = 0,
405         ARM_SMMU_DOMAIN_S2,
406         ARM_SMMU_DOMAIN_NESTED,
407 };
408 
409 struct arm_smmu_domain {
410         struct arm_smmu_device          *smmu;
411         struct io_pgtable_ops           *pgtbl_ops;
412         spinlock_t                      pgtbl_lock;
413         struct arm_smmu_cfg             cfg;
414         enum arm_smmu_domain_stage      stage;
415         struct mutex                    init_mutex; /* Protects smmu pointer */
416         struct iommu_domain             domain;
417 };
418 
419 struct arm_smmu_option_prop {
420         u32 opt;
421         const char *prop;
422 };
423 
424 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
425 
426 static bool using_legacy_binding, using_generic_binding;
427 
428 static struct arm_smmu_option_prop arm_smmu_options[] = {
429         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
430         { 0, NULL},
431 };
432 
433 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
434 {
435         return container_of(dom, struct arm_smmu_domain, domain);
436 }
437 
438 static void parse_driver_options(struct arm_smmu_device *smmu)
439 {
440         int i = 0;
441 
442         do {
443                 if (of_property_read_bool(smmu->dev->of_node,
444                                                 arm_smmu_options[i].prop)) {
445                         smmu->options |= arm_smmu_options[i].opt;
446                         dev_notice(smmu->dev, "option %s\n",
447                                 arm_smmu_options[i].prop);
448                 }
449         } while (arm_smmu_options[++i].opt);
450 }
451 
452 static struct device_node *dev_get_dev_node(struct device *dev)
453 {
454         if (dev_is_pci(dev)) {
455                 struct pci_bus *bus = to_pci_dev(dev)->bus;
456 
457                 while (!pci_is_root_bus(bus))
458                         bus = bus->parent;
459                 return of_node_get(bus->bridge->parent->of_node);
460         }
461 
462         return of_node_get(dev->of_node);
463 }
464 
465 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
466 {
467         *((__be32 *)data) = cpu_to_be32(alias);
468         return 0; /* Continue walking */
469 }
470 
471 static int __find_legacy_master_phandle(struct device *dev, void *data)
472 {
473         struct of_phandle_iterator *it = *(void **)data;
474         struct device_node *np = it->node;
475         int err;
476 
477         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
478                             "#stream-id-cells", 0)
479                 if (it->node == np) {
480                         *(void **)data = dev;
481                         return 1;
482                 }
483         it->node = np;
484         return err == -ENOENT ? 0 : err;
485 }
486 
487 static struct platform_driver arm_smmu_driver;
488 static struct iommu_ops arm_smmu_ops;
489 
490 static int arm_smmu_register_legacy_master(struct device *dev,
491                                            struct arm_smmu_device **smmu)
492 {
493         struct device *smmu_dev;
494         struct device_node *np;
495         struct of_phandle_iterator it;
496         void *data = &it;
497         u32 *sids;
498         __be32 pci_sid;
499         int err;
500 
501         np = dev_get_dev_node(dev);
502         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
503                 of_node_put(np);
504                 return -ENODEV;
505         }
506 
507         it.node = np;
508         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
509                                      __find_legacy_master_phandle);
510         smmu_dev = data;
511         of_node_put(np);
512         if (err == 0)
513                 return -ENODEV;
514         if (err < 0)
515                 return err;
516 
517         if (dev_is_pci(dev)) {
518                 /* "mmu-masters" assumes Stream ID == Requester ID */
519                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
520                                        &pci_sid);
521                 it.cur = &pci_sid;
522                 it.cur_count = 1;
523         }
524 
525         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
526                                 &arm_smmu_ops);
527         if (err)
528                 return err;
529 
530         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
531         if (!sids)
532                 return -ENOMEM;
533 
534         *smmu = dev_get_drvdata(smmu_dev);
535         of_phandle_iterator_args(&it, sids, it.cur_count);
536         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
537         kfree(sids);
538         return err;
539 }
540 
541 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
542 {
543         int idx;
544 
545         do {
546                 idx = find_next_zero_bit(map, end, start);
547                 if (idx == end)
548                         return -ENOSPC;
549         } while (test_and_set_bit(idx, map));
550 
551         return idx;
552 }
553 
554 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
555 {
556         clear_bit(idx, map);
557 }
558 
559 /* Wait for any pending TLB invalidations to complete */
560 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
561 {
562         int count = 0;
563         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
564 
565         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
566         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
567                & sTLBGSTATUS_GSACTIVE) {
568                 cpu_relax();
569                 if (++count == TLB_LOOP_TIMEOUT) {
570                         dev_err_ratelimited(smmu->dev,
571                         "TLB sync timed out -- SMMU may be deadlocked\n");
572                         return;
573                 }
574                 udelay(1);
575         }
576 }
577 
578 static void arm_smmu_tlb_sync(void *cookie)
579 {
580         struct arm_smmu_domain *smmu_domain = cookie;
581         __arm_smmu_tlb_sync(smmu_domain->smmu);
582 }
583 
584 static void arm_smmu_tlb_inv_context(void *cookie)
585 {
586         struct arm_smmu_domain *smmu_domain = cookie;
587         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
588         struct arm_smmu_device *smmu = smmu_domain->smmu;
589         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
590         void __iomem *base;
591 
592         if (stage1) {
593                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
594                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
595                                base + ARM_SMMU_CB_S1_TLBIASID);
596         } else {
597                 base = ARM_SMMU_GR0(smmu);
598                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
599                                base + ARM_SMMU_GR0_TLBIVMID);
600         }
601 
602         __arm_smmu_tlb_sync(smmu);
603 }
604 
605 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
606                                           size_t granule, bool leaf, void *cookie)
607 {
608         struct arm_smmu_domain *smmu_domain = cookie;
609         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
610         struct arm_smmu_device *smmu = smmu_domain->smmu;
611         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
612         void __iomem *reg;
613 
614         if (stage1) {
615                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
616                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
617 
618                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
619                         iova &= ~12UL;
620                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
621                         do {
622                                 writel_relaxed(iova, reg);
623                                 iova += granule;
624                         } while (size -= granule);
625                 } else {
626                         iova >>= 12;
627                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
628                         do {
629                                 writeq_relaxed(iova, reg);
630                                 iova += granule >> 12;
631                         } while (size -= granule);
632                 }
633         } else if (smmu->version == ARM_SMMU_V2) {
634                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
635                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
636                               ARM_SMMU_CB_S2_TLBIIPAS2;
637                 iova >>= 12;
638                 do {
639                         smmu_write_atomic_lq(iova, reg);
640                         iova += granule >> 12;
641                 } while (size -= granule);
642         } else {
643                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
644                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
645         }
646 }
647 
648 static const struct iommu_gather_ops arm_smmu_gather_ops = {
649         .tlb_flush_all  = arm_smmu_tlb_inv_context,
650         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
651         .tlb_sync       = arm_smmu_tlb_sync,
652 };
653 
654 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
655 {
656         u32 fsr, fsynr;
657         unsigned long iova;
658         struct iommu_domain *domain = dev;
659         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
660         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
661         struct arm_smmu_device *smmu = smmu_domain->smmu;
662         void __iomem *cb_base;
663 
664         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
665         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
666 
667         if (!(fsr & FSR_FAULT))
668                 return IRQ_NONE;
669 
670         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
671         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
672 
673         dev_err_ratelimited(smmu->dev,
674         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
675                             fsr, iova, fsynr, cfg->cbndx);
676 
677         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
678         return IRQ_HANDLED;
679 }
680 
681 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
682 {
683         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
684         struct arm_smmu_device *smmu = dev;
685         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
686 
687         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
688         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
689         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
690         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
691 
692         if (!gfsr)
693                 return IRQ_NONE;
694 
695         dev_err_ratelimited(smmu->dev,
696                 "Unexpected global fault, this could be serious\n");
697         dev_err_ratelimited(smmu->dev,
698                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
699                 gfsr, gfsynr0, gfsynr1, gfsynr2);
700 
701         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
702         return IRQ_HANDLED;
703 }
704 
705 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
706                                        struct io_pgtable_cfg *pgtbl_cfg)
707 {
708         u32 reg, reg2;
709         u64 reg64;
710         bool stage1;
711         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
712         struct arm_smmu_device *smmu = smmu_domain->smmu;
713         void __iomem *cb_base, *gr1_base;
714 
715         gr1_base = ARM_SMMU_GR1(smmu);
716         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
717         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
718 
719         if (smmu->version > ARM_SMMU_V1) {
720                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
721                         reg = CBA2R_RW64_64BIT;
722                 else
723                         reg = CBA2R_RW64_32BIT;
724                 /* 16-bit VMIDs live in CBA2R */
725                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
726                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
727 
728                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
729         }
730 
731         /* CBAR */
732         reg = cfg->cbar;
733         if (smmu->version < ARM_SMMU_V2)
734                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
735 
736         /*
737          * Use the weakest shareability/memory types, so they are
738          * overridden by the ttbcr/pte.
739          */
740         if (stage1) {
741                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
742                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
743         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
744                 /* 8-bit VMIDs live in CBAR */
745                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
746         }
747         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
748 
749         /* TTBRs */
750         if (stage1) {
751                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
752 
753                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
754                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
755                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
756                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
757                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
758                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
759                 } else {
760                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
761                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
762                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
763                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
764                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
765                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
766                 }
767         } else {
768                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
769                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
770         }
771 
772         /* TTBCR */
773         if (stage1) {
774                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
775                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
776                         reg2 = 0;
777                 } else {
778                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
779                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
780                         reg2 |= TTBCR2_SEP_UPSTREAM;
781                 }
782                 if (smmu->version > ARM_SMMU_V1)
783                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
784         } else {
785                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
786         }
787         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
788 
789         /* MAIRs (stage-1 only) */
790         if (stage1) {
791                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
792                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
793                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
794                 } else {
795                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
796                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
797                 }
798                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
799                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
800         }
801 
802         /* SCTLR */
803         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
804         if (stage1)
805                 reg |= SCTLR_S1_ASIDPNE;
806 #ifdef __BIG_ENDIAN
807         reg |= SCTLR_E;
808 #endif
809         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
810 }
811 
812 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
813                                         struct arm_smmu_device *smmu)
814 {
815         int irq, start, ret = 0;
816         unsigned long ias, oas;
817         struct io_pgtable_ops *pgtbl_ops;
818         struct io_pgtable_cfg pgtbl_cfg;
819         enum io_pgtable_fmt fmt;
820         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
821         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
822 
823         mutex_lock(&smmu_domain->init_mutex);
824         if (smmu_domain->smmu)
825                 goto out_unlock;
826 
827         /*
828          * Mapping the requested stage onto what we support is surprisingly
829          * complicated, mainly because the spec allows S1+S2 SMMUs without
830          * support for nested translation. That means we end up with the
831          * following table:
832          *
833          * Requested        Supported        Actual
834          *     S1               N              S1
835          *     S1             S1+S2            S1
836          *     S1               S2             S2
837          *     S1               S1             S1
838          *     N                N              N
839          *     N              S1+S2            S2
840          *     N                S2             S2
841          *     N                S1             S1
842          *
843          * Note that you can't actually request stage-2 mappings.
844          */
845         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
846                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
847         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
848                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
849 
850         /*
851          * Choosing a suitable context format is even more fiddly. Until we
852          * grow some way for the caller to express a preference, and/or move
853          * the decision into the io-pgtable code where it arguably belongs,
854          * just aim for the closest thing to the rest of the system, and hope
855          * that the hardware isn't esoteric enough that we can't assume AArch64
856          * support to be a superset of AArch32 support...
857          */
858         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
859                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
860         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
861             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
862             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
863             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
864                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
865         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
866             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
867                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
868                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
869                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
870 
871         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
872                 ret = -EINVAL;
873                 goto out_unlock;
874         }
875 
876         switch (smmu_domain->stage) {
877         case ARM_SMMU_DOMAIN_S1:
878                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
879                 start = smmu->num_s2_context_banks;
880                 ias = smmu->va_size;
881                 oas = smmu->ipa_size;
882                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
883                         fmt = ARM_64_LPAE_S1;
884                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
885                         fmt = ARM_32_LPAE_S1;
886                         ias = min(ias, 32UL);
887                         oas = min(oas, 40UL);
888                 } else {
889                         fmt = ARM_V7S;
890                         ias = min(ias, 32UL);
891                         oas = min(oas, 32UL);
892                 }
893                 break;
894         case ARM_SMMU_DOMAIN_NESTED:
895                 /*
896                  * We will likely want to change this if/when KVM gets
897                  * involved.
898                  */
899         case ARM_SMMU_DOMAIN_S2:
900                 cfg->cbar = CBAR_TYPE_S2_TRANS;
901                 start = 0;
902                 ias = smmu->ipa_size;
903                 oas = smmu->pa_size;
904                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
905                         fmt = ARM_64_LPAE_S2;
906                 } else {
907                         fmt = ARM_32_LPAE_S2;
908                         ias = min(ias, 40UL);
909                         oas = min(oas, 40UL);
910                 }
911                 break;
912         default:
913                 ret = -EINVAL;
914                 goto out_unlock;
915         }
916 
917         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
918                                       smmu->num_context_banks);
919         if (ret < 0)
920                 goto out_unlock;
921 
922         cfg->cbndx = ret;
923         if (smmu->version < ARM_SMMU_V2) {
924                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
925                 cfg->irptndx %= smmu->num_context_irqs;
926         } else {
927                 cfg->irptndx = cfg->cbndx;
928         }
929 
930         pgtbl_cfg = (struct io_pgtable_cfg) {
931                 .pgsize_bitmap  = smmu->pgsize_bitmap,
932                 .ias            = ias,
933                 .oas            = oas,
934                 .tlb            = &arm_smmu_gather_ops,
935                 .iommu_dev      = smmu->dev,
936         };
937 
938         smmu_domain->smmu = smmu;
939         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
940         if (!pgtbl_ops) {
941                 ret = -ENOMEM;
942                 goto out_clear_smmu;
943         }
944 
945         /* Update the domain's page sizes to reflect the page table format */
946         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
947         domain->geometry.aperture_end = (1UL << ias) - 1;
948         domain->geometry.force_aperture = true;
949 
950         /* Initialise the context bank with our page table cfg */
951         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
952 
953         /*
954          * Request context fault interrupt. Do this last to avoid the
955          * handler seeing a half-initialised domain state.
956          */
957         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
958         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
959                                IRQF_SHARED, "arm-smmu-context-fault", domain);
960         if (ret < 0) {
961                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
962                         cfg->irptndx, irq);
963                 cfg->irptndx = INVALID_IRPTNDX;
964         }
965 
966         mutex_unlock(&smmu_domain->init_mutex);
967 
968         /* Publish page table ops for map/unmap */
969         smmu_domain->pgtbl_ops = pgtbl_ops;
970         return 0;
971 
972 out_clear_smmu:
973         smmu_domain->smmu = NULL;
974 out_unlock:
975         mutex_unlock(&smmu_domain->init_mutex);
976         return ret;
977 }
978 
979 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
980 {
981         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
982         struct arm_smmu_device *smmu = smmu_domain->smmu;
983         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
984         void __iomem *cb_base;
985         int irq;
986 
987         if (!smmu)
988                 return;
989 
990         /*
991          * Disable the context bank and free the page tables before freeing
992          * it.
993          */
994         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
995         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
996 
997         if (cfg->irptndx != INVALID_IRPTNDX) {
998                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
999                 devm_free_irq(smmu->dev, irq, domain);
1000         }
1001 
1002         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1003         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1004 }
1005 
1006 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1007 {
1008         struct arm_smmu_domain *smmu_domain;
1009 
1010         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1011                 return NULL;
1012         /*
1013          * Allocate the domain and initialise some of its data structures.
1014          * We can't really do anything meaningful until we've added a
1015          * master.
1016          */
1017         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1018         if (!smmu_domain)
1019                 return NULL;
1020 
1021         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1022             iommu_get_dma_cookie(&smmu_domain->domain))) {
1023                 kfree(smmu_domain);
1024                 return NULL;
1025         }
1026 
1027         mutex_init(&smmu_domain->init_mutex);
1028         spin_lock_init(&smmu_domain->pgtbl_lock);
1029 
1030         return &smmu_domain->domain;
1031 }
1032 
1033 static void arm_smmu_domain_free(struct iommu_domain *domain)
1034 {
1035         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1036 
1037         /*
1038          * Free the domain resources. We assume that all devices have
1039          * already been detached.
1040          */
1041         iommu_put_dma_cookie(domain);
1042         arm_smmu_destroy_domain_context(domain);
1043         kfree(smmu_domain);
1044 }
1045 
1046 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1047 {
1048         struct arm_smmu_smr *smr = smmu->smrs + idx;
1049         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1050 
1051         if (smr->valid)
1052                 reg |= SMR_VALID;
1053         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1054 }
1055 
1056 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1057 {
1058         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1059         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1060                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1061                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1062 
1063         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1064 }
1065 
1066 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1067 {
1068         arm_smmu_write_s2cr(smmu, idx);
1069         if (smmu->smrs)
1070                 arm_smmu_write_smr(smmu, idx);
1071 }
1072 
1073 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1074 {
1075         struct arm_smmu_smr *smrs = smmu->smrs;
1076         int i, free_idx = -ENOSPC;
1077 
1078         /* Stream indexing is blissfully easy */
1079         if (!smrs)
1080                 return id;
1081 
1082         /* Validating SMRs is... less so */
1083         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1084                 if (!smrs[i].valid) {
1085                         /*
1086                          * Note the first free entry we come across, which
1087                          * we'll claim in the end if nothing else matches.
1088                          */
1089                         if (free_idx < 0)
1090                                 free_idx = i;
1091                         continue;
1092                 }
1093                 /*
1094                  * If the new entry is _entirely_ matched by an existing entry,
1095                  * then reuse that, with the guarantee that there also cannot
1096                  * be any subsequent conflicting entries. In normal use we'd
1097                  * expect simply identical entries for this case, but there's
1098                  * no harm in accommodating the generalisation.
1099                  */
1100                 if ((mask & smrs[i].mask) == mask &&
1101                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1102                         return i;
1103                 /*
1104                  * If the new entry has any other overlap with an existing one,
1105                  * though, then there always exists at least one stream ID
1106                  * which would cause a conflict, and we can't allow that risk.
1107                  */
1108                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1109                         return -EINVAL;
1110         }
1111 
1112         return free_idx;
1113 }
1114 
1115 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1116 {
1117         if (--smmu->s2crs[idx].count)
1118                 return false;
1119 
1120         smmu->s2crs[idx] = s2cr_init_val;
1121         if (smmu->smrs)
1122                 smmu->smrs[idx].valid = false;
1123 
1124         return true;
1125 }
1126 
1127 static int arm_smmu_master_alloc_smes(struct device *dev)
1128 {
1129         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1130         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1131         struct arm_smmu_device *smmu = cfg->smmu;
1132         struct arm_smmu_smr *smrs = smmu->smrs;
1133         struct iommu_group *group;
1134         int i, idx, ret;
1135 
1136         mutex_lock(&smmu->stream_map_mutex);
1137         /* Figure out a viable stream map entry allocation */
1138         for_each_cfg_sme(fwspec, i, idx) {
1139                 u16 sid = fwspec->ids[i];
1140                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1141 
1142                 if (idx != INVALID_SMENDX) {
1143                         ret = -EEXIST;
1144                         goto out_err;
1145                 }
1146 
1147                 ret = arm_smmu_find_sme(smmu, sid, mask);
1148                 if (ret < 0)
1149                         goto out_err;
1150 
1151                 idx = ret;
1152                 if (smrs && smmu->s2crs[idx].count == 0) {
1153                         smrs[idx].id = sid;
1154                         smrs[idx].mask = mask;
1155                         smrs[idx].valid = true;
1156                 }
1157                 smmu->s2crs[idx].count++;
1158                 cfg->smendx[i] = (s16)idx;
1159         }
1160 
1161         group = iommu_group_get_for_dev(dev);
1162         if (!group)
1163                 group = ERR_PTR(-ENOMEM);
1164         if (IS_ERR(group)) {
1165                 ret = PTR_ERR(group);
1166                 goto out_err;
1167         }
1168         iommu_group_put(group);
1169 
1170         /* It worked! Now, poke the actual hardware */
1171         for_each_cfg_sme(fwspec, i, idx) {
1172                 arm_smmu_write_sme(smmu, idx);
1173                 smmu->s2crs[idx].group = group;
1174         }
1175 
1176         mutex_unlock(&smmu->stream_map_mutex);
1177         return 0;
1178 
1179 out_err:
1180         while (i--) {
1181                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1182                 cfg->smendx[i] = INVALID_SMENDX;
1183         }
1184         mutex_unlock(&smmu->stream_map_mutex);
1185         return ret;
1186 }
1187 
1188 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1189 {
1190         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1191         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1192         int i, idx;
1193 
1194         mutex_lock(&smmu->stream_map_mutex);
1195         for_each_cfg_sme(fwspec, i, idx) {
1196                 if (arm_smmu_free_sme(smmu, idx))
1197                         arm_smmu_write_sme(smmu, idx);
1198                 cfg->smendx[i] = INVALID_SMENDX;
1199         }
1200         mutex_unlock(&smmu->stream_map_mutex);
1201 }
1202 
1203 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1204                                       struct iommu_fwspec *fwspec)
1205 {
1206         struct arm_smmu_device *smmu = smmu_domain->smmu;
1207         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1208         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1209         u8 cbndx = smmu_domain->cfg.cbndx;
1210         int i, idx;
1211 
1212         for_each_cfg_sme(fwspec, i, idx) {
1213                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1214                         continue;
1215 
1216                 s2cr[idx].type = type;
1217                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1218                 s2cr[idx].cbndx = cbndx;
1219                 arm_smmu_write_s2cr(smmu, idx);
1220         }
1221         return 0;
1222 }
1223 
1224 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1225 {
1226         int ret;
1227         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1228         struct arm_smmu_device *smmu;
1229         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1230 
1231         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1232                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1233                 return -ENXIO;
1234         }
1235 
1236         /*
1237          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1238          * domains between of_xlate() and add_device() - we have no way to cope
1239          * with that, so until ARM gets converted to rely on groups and default
1240          * domains, just say no (but more politely than by dereferencing NULL).
1241          * This should be at least a WARN_ON once that's sorted.
1242          */
1243         if (!fwspec->iommu_priv)
1244                 return -ENODEV;
1245 
1246         smmu = fwspec_smmu(fwspec);
1247         /* Ensure that the domain is finalised */
1248         ret = arm_smmu_init_domain_context(domain, smmu);
1249         if (ret < 0)
1250                 return ret;
1251 
1252         /*
1253          * Sanity check the domain. We don't support domains across
1254          * different SMMUs.
1255          */
1256         if (smmu_domain->smmu != smmu) {
1257                 dev_err(dev,
1258                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1259                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1260                 return -EINVAL;
1261         }
1262 
1263         /* Looks ok, so add the device to the domain */
1264         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1265 }
1266 
1267 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1268                         phys_addr_t paddr, size_t size, int prot)
1269 {
1270         int ret;
1271         unsigned long flags;
1272         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1273         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1274 
1275         if (!ops)
1276                 return -ENODEV;
1277 
1278         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1279         ret = ops->map(ops, iova, paddr, size, prot);
1280         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1281         return ret;
1282 }
1283 
1284 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1285                              size_t size)
1286 {
1287         size_t ret;
1288         unsigned long flags;
1289         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1290         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1291 
1292         if (!ops)
1293                 return 0;
1294 
1295         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1296         ret = ops->unmap(ops, iova, size);
1297         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1298         return ret;
1299 }
1300 
1301 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1302                                               dma_addr_t iova)
1303 {
1304         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1305         struct arm_smmu_device *smmu = smmu_domain->smmu;
1306         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1307         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1308         struct device *dev = smmu->dev;
1309         void __iomem *cb_base;
1310         u32 tmp;
1311         u64 phys;
1312         unsigned long va;
1313 
1314         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1315 
1316         /* ATS1 registers can only be written atomically */
1317         va = iova & ~0xfffUL;
1318         if (smmu->version == ARM_SMMU_V2)
1319                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1320         else /* Register is only 32-bit in v1 */
1321                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1322 
1323         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1324                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1325                 dev_err(dev,
1326                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1327                         &iova);
1328                 return ops->iova_to_phys(ops, iova);
1329         }
1330 
1331         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1332         if (phys & CB_PAR_F) {
1333                 dev_err(dev, "translation fault!\n");
1334                 dev_err(dev, "PAR = 0x%llx\n", phys);
1335                 return 0;
1336         }
1337 
1338         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1339 }
1340 
1341 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1342                                         dma_addr_t iova)
1343 {
1344         phys_addr_t ret;
1345         unsigned long flags;
1346         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1347         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1348 
1349         if (!ops)
1350                 return 0;
1351 
1352         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1353         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1354                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1355                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1356         } else {
1357                 ret = ops->iova_to_phys(ops, iova);
1358         }
1359 
1360         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1361 
1362         return ret;
1363 }
1364 
1365 static bool arm_smmu_capable(enum iommu_cap cap)
1366 {
1367         switch (cap) {
1368         case IOMMU_CAP_CACHE_COHERENCY:
1369                 /*
1370                  * Return true here as the SMMU can always send out coherent
1371                  * requests.
1372                  */
1373                 return true;
1374         case IOMMU_CAP_INTR_REMAP:
1375                 return true; /* MSIs are just memory writes */
1376         case IOMMU_CAP_NOEXEC:
1377                 return true;
1378         default:
1379                 return false;
1380         }
1381 }
1382 
1383 static int arm_smmu_match_node(struct device *dev, void *data)
1384 {
1385         return dev->fwnode == data;
1386 }
1387 
1388 static
1389 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1390 {
1391         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1392                                                 fwnode, arm_smmu_match_node);
1393         put_device(dev);
1394         return dev ? dev_get_drvdata(dev) : NULL;
1395 }
1396 
1397 static int arm_smmu_add_device(struct device *dev)
1398 {
1399         struct arm_smmu_device *smmu;
1400         struct arm_smmu_master_cfg *cfg;
1401         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1402         int i, ret;
1403 
1404         if (using_legacy_binding) {
1405                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1406                 fwspec = dev->iommu_fwspec;
1407                 if (ret)
1408                         goto out_free;
1409         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1410                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1411         } else {
1412                 return -ENODEV;
1413         }
1414 
1415         ret = -EINVAL;
1416         for (i = 0; i < fwspec->num_ids; i++) {
1417                 u16 sid = fwspec->ids[i];
1418                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1419 
1420                 if (sid & ~smmu->streamid_mask) {
1421                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1422                                 sid, smmu->streamid_mask);
1423                         goto out_free;
1424                 }
1425                 if (mask & ~smmu->smr_mask_mask) {
1426                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1427                                 sid, smmu->smr_mask_mask);
1428                         goto out_free;
1429                 }
1430         }
1431 
1432         ret = -ENOMEM;
1433         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1434                       GFP_KERNEL);
1435         if (!cfg)
1436                 goto out_free;
1437 
1438         cfg->smmu = smmu;
1439         fwspec->iommu_priv = cfg;
1440         while (i--)
1441                 cfg->smendx[i] = INVALID_SMENDX;
1442 
1443         ret = arm_smmu_master_alloc_smes(dev);
1444         if (ret)
1445                 goto out_free;
1446 
1447         return 0;
1448 
1449 out_free:
1450         if (fwspec)
1451                 kfree(fwspec->iommu_priv);
1452         iommu_fwspec_free(dev);
1453         return ret;
1454 }
1455 
1456 static void arm_smmu_remove_device(struct device *dev)
1457 {
1458         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1459 
1460         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1461                 return;
1462 
1463         arm_smmu_master_free_smes(fwspec);
1464         iommu_group_remove_device(dev);
1465         kfree(fwspec->iommu_priv);
1466         iommu_fwspec_free(dev);
1467 }
1468 
1469 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1470 {
1471         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1472         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1473         struct iommu_group *group = NULL;
1474         int i, idx;
1475 
1476         for_each_cfg_sme(fwspec, i, idx) {
1477                 if (group && smmu->s2crs[idx].group &&
1478                     group != smmu->s2crs[idx].group)
1479                         return ERR_PTR(-EINVAL);
1480 
1481                 group = smmu->s2crs[idx].group;
1482         }
1483 
1484         if (group)
1485                 return iommu_group_ref_get(group);
1486 
1487         if (dev_is_pci(dev))
1488                 group = pci_device_group(dev);
1489         else
1490                 group = generic_device_group(dev);
1491 
1492         return group;
1493 }
1494 
1495 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1496                                     enum iommu_attr attr, void *data)
1497 {
1498         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1499 
1500         switch (attr) {
1501         case DOMAIN_ATTR_NESTING:
1502                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1503                 return 0;
1504         default:
1505                 return -ENODEV;
1506         }
1507 }
1508 
1509 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1510                                     enum iommu_attr attr, void *data)
1511 {
1512         int ret = 0;
1513         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1514 
1515         mutex_lock(&smmu_domain->init_mutex);
1516 
1517         switch (attr) {
1518         case DOMAIN_ATTR_NESTING:
1519                 if (smmu_domain->smmu) {
1520                         ret = -EPERM;
1521                         goto out_unlock;
1522                 }
1523 
1524                 if (*(int *)data)
1525                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1526                 else
1527                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1528 
1529                 break;
1530         default:
1531                 ret = -ENODEV;
1532         }
1533 
1534 out_unlock:
1535         mutex_unlock(&smmu_domain->init_mutex);
1536         return ret;
1537 }
1538 
1539 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1540 {
1541         u32 fwid = 0;
1542 
1543         if (args->args_count > 0)
1544                 fwid |= (u16)args->args[0];
1545 
1546         if (args->args_count > 1)
1547                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1548 
1549         return iommu_fwspec_add_ids(dev, &fwid, 1);
1550 }
1551 
1552 static struct iommu_ops arm_smmu_ops = {
1553         .capable                = arm_smmu_capable,
1554         .domain_alloc           = arm_smmu_domain_alloc,
1555         .domain_free            = arm_smmu_domain_free,
1556         .attach_dev             = arm_smmu_attach_dev,
1557         .map                    = arm_smmu_map,
1558         .unmap                  = arm_smmu_unmap,
1559         .map_sg                 = default_iommu_map_sg,
1560         .iova_to_phys           = arm_smmu_iova_to_phys,
1561         .add_device             = arm_smmu_add_device,
1562         .remove_device          = arm_smmu_remove_device,
1563         .device_group           = arm_smmu_device_group,
1564         .domain_get_attr        = arm_smmu_domain_get_attr,
1565         .domain_set_attr        = arm_smmu_domain_set_attr,
1566         .of_xlate               = arm_smmu_of_xlate,
1567         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1568 };
1569 
1570 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1571 {
1572         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1573         void __iomem *cb_base;
1574         int i;
1575         u32 reg, major;
1576 
1577         /* clear global FSR */
1578         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1579         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1580 
1581         /*
1582          * Reset stream mapping groups: Initial values mark all SMRn as
1583          * invalid and all S2CRn as bypass unless overridden.
1584          */
1585         for (i = 0; i < smmu->num_mapping_groups; ++i)
1586                 arm_smmu_write_sme(smmu, i);
1587 
1588         if (smmu->model == ARM_MMU500) {
1589                 /*
1590                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1591                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1592                  * bit is only present in MMU-500r2 onwards.
1593                  */
1594                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1595                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1596                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1597                 if (major >= 2)
1598                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1599                 /*
1600                  * Allow unmatched Stream IDs to allocate bypass
1601                  * TLB entries for reduced latency.
1602                  */
1603                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1604                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1605         }
1606 
1607         /* Make sure all context banks are disabled and clear CB_FSR  */
1608         for (i = 0; i < smmu->num_context_banks; ++i) {
1609                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1610                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1611                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1612                 /*
1613                  * Disable MMU-500's not-particularly-beneficial next-page
1614                  * prefetcher for the sake of errata #841119 and #826419.
1615                  */
1616                 if (smmu->model == ARM_MMU500) {
1617                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1618                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1619                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1620                 }
1621         }
1622 
1623         /* Invalidate the TLB, just in case */
1624         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1625         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1626 
1627         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1628 
1629         /* Enable fault reporting */
1630         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1631 
1632         /* Disable TLB broadcasting. */
1633         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1634 
1635         /* Enable client access, handling unmatched streams as appropriate */
1636         reg &= ~sCR0_CLIENTPD;
1637         if (disable_bypass)
1638                 reg |= sCR0_USFCFG;
1639         else
1640                 reg &= ~sCR0_USFCFG;
1641 
1642         /* Disable forced broadcasting */
1643         reg &= ~sCR0_FB;
1644 
1645         /* Don't upgrade barriers */
1646         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1647 
1648         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1649                 reg |= sCR0_VMID16EN;
1650 
1651         /* Push the button */
1652         __arm_smmu_tlb_sync(smmu);
1653         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1654 }
1655 
1656 static int arm_smmu_id_size_to_bits(int size)
1657 {
1658         switch (size) {
1659         case 0:
1660                 return 32;
1661         case 1:
1662                 return 36;
1663         case 2:
1664                 return 40;
1665         case 3:
1666                 return 42;
1667         case 4:
1668                 return 44;
1669         case 5:
1670         default:
1671                 return 48;
1672         }
1673 }
1674 
1675 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1676 {
1677         unsigned long size;
1678         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1679         u32 id;
1680         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1681         int i;
1682 
1683         dev_notice(smmu->dev, "probing hardware configuration...\n");
1684         dev_notice(smmu->dev, "SMMUv%d with:\n",
1685                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1686 
1687         /* ID0 */
1688         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1689 
1690         /* Restrict available stages based on module parameter */
1691         if (force_stage == 1)
1692                 id &= ~(ID0_S2TS | ID0_NTS);
1693         else if (force_stage == 2)
1694                 id &= ~(ID0_S1TS | ID0_NTS);
1695 
1696         if (id & ID0_S1TS) {
1697                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1698                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1699         }
1700 
1701         if (id & ID0_S2TS) {
1702                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1703                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1704         }
1705 
1706         if (id & ID0_NTS) {
1707                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1708                 dev_notice(smmu->dev, "\tnested translation\n");
1709         }
1710 
1711         if (!(smmu->features &
1712                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1713                 dev_err(smmu->dev, "\tno translation support!\n");
1714                 return -ENODEV;
1715         }
1716 
1717         if ((id & ID0_S1TS) &&
1718                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1719                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1720                 dev_notice(smmu->dev, "\taddress translation ops\n");
1721         }
1722 
1723         /*
1724          * In order for DMA API calls to work properly, we must defer to what
1725          * the FW says about coherency, regardless of what the hardware claims.
1726          * Fortunately, this also opens up a workaround for systems where the
1727          * ID register value has ended up configured incorrectly.
1728          */
1729         cttw_reg = !!(id & ID0_CTTW);
1730         if (cttw_fw || cttw_reg)
1731                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1732                            cttw_fw ? "" : "non-");
1733         if (cttw_fw != cttw_reg)
1734                 dev_notice(smmu->dev,
1735                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1736 
1737         /* Max. number of entries we have for stream matching/indexing */
1738         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1739         smmu->streamid_mask = size - 1;
1740         if (id & ID0_SMS) {
1741                 u32 smr;
1742 
1743                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1744                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1745                 if (size == 0) {
1746                         dev_err(smmu->dev,
1747                                 "stream-matching supported, but no SMRs present!\n");
1748                         return -ENODEV;
1749                 }
1750 
1751                 /*
1752                  * SMR.ID bits may not be preserved if the corresponding MASK
1753                  * bits are set, so check each one separately. We can reject
1754                  * masters later if they try to claim IDs outside these masks.
1755                  */
1756                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1757                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1758                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1759                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1760 
1761                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1762                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1763                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1764                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1765 
1766                 /* Zero-initialised to mark as invalid */
1767                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1768                                           GFP_KERNEL);
1769                 if (!smmu->smrs)
1770                         return -ENOMEM;
1771 
1772                 dev_notice(smmu->dev,
1773                            "\tstream matching with %lu register groups, mask 0x%x",
1774                            size, smmu->smr_mask_mask);
1775         }
1776         /* s2cr->type == 0 means translation, so initialise explicitly */
1777         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1778                                          GFP_KERNEL);
1779         if (!smmu->s2crs)
1780                 return -ENOMEM;
1781         for (i = 0; i < size; i++)
1782                 smmu->s2crs[i] = s2cr_init_val;
1783 
1784         smmu->num_mapping_groups = size;
1785         mutex_init(&smmu->stream_map_mutex);
1786 
1787         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1788                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1789                 if (!(id & ID0_PTFS_NO_AARCH32S))
1790                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1791         }
1792 
1793         /* ID1 */
1794         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1795         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1796 
1797         /* Check for size mismatch of SMMU address space from mapped region */
1798         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1799         size *= 2 << smmu->pgshift;
1800         if (smmu->size != size)
1801                 dev_warn(smmu->dev,
1802                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1803                         size, smmu->size);
1804 
1805         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1806         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1807         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1808                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1809                 return -ENODEV;
1810         }
1811         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1812                    smmu->num_context_banks, smmu->num_s2_context_banks);
1813         /*
1814          * Cavium CN88xx erratum #27704.
1815          * Ensure ASID and VMID allocation is unique across all SMMUs in
1816          * the system.
1817          */
1818         if (smmu->model == CAVIUM_SMMUV2) {
1819                 smmu->cavium_id_base =
1820                         atomic_add_return(smmu->num_context_banks,
1821                                           &cavium_smmu_context_count);
1822                 smmu->cavium_id_base -= smmu->num_context_banks;
1823         }
1824 
1825         /* ID2 */
1826         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1827         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1828         smmu->ipa_size = size;
1829 
1830         /* The output mask is also applied for bypass */
1831         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1832         smmu->pa_size = size;
1833 
1834         if (id & ID2_VMID16)
1835                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1836 
1837         /*
1838          * What the page table walker can address actually depends on which
1839          * descriptor format is in use, but since a) we don't know that yet,
1840          * and b) it can vary per context bank, this will have to do...
1841          */
1842         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1843                 dev_warn(smmu->dev,
1844                          "failed to set DMA mask for table walker\n");
1845 
1846         if (smmu->version < ARM_SMMU_V2) {
1847                 smmu->va_size = smmu->ipa_size;
1848                 if (smmu->version == ARM_SMMU_V1_64K)
1849                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1850         } else {
1851                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1852                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1853                 if (id & ID2_PTFS_4K)
1854                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1855                 if (id & ID2_PTFS_16K)
1856                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1857                 if (id & ID2_PTFS_64K)
1858                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1859         }
1860 
1861         /* Now we've corralled the various formats, what'll it do? */
1862         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1863                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1864         if (smmu->features &
1865             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1866                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1867         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1868                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1869         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1870                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1871 
1872         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1873                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1874         else
1875                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1876         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1877                    smmu->pgsize_bitmap);
1878 
1879 
1880         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1881                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1882                            smmu->va_size, smmu->ipa_size);
1883 
1884         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1885                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1886                            smmu->ipa_size, smmu->pa_size);
1887 
1888         return 0;
1889 }
1890 
1891 struct arm_smmu_match_data {
1892         enum arm_smmu_arch_version version;
1893         enum arm_smmu_implementation model;
1894 };
1895 
1896 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1897 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1898 
1899 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1900 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1901 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1902 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1903 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1904 
1905 static const struct of_device_id arm_smmu_of_match[] = {
1906         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1907         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1908         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1909         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1910         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1911         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1912         { },
1913 };
1914 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1915 
1916 #ifdef CONFIG_ACPI
1917 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1918 {
1919         int ret = 0;
1920 
1921         switch (model) {
1922         case ACPI_IORT_SMMU_V1:
1923         case ACPI_IORT_SMMU_CORELINK_MMU400:
1924                 smmu->version = ARM_SMMU_V1;
1925                 smmu->model = GENERIC_SMMU;
1926                 break;
1927         case ACPI_IORT_SMMU_V2:
1928                 smmu->version = ARM_SMMU_V2;
1929                 smmu->model = GENERIC_SMMU;
1930                 break;
1931         case ACPI_IORT_SMMU_CORELINK_MMU500:
1932                 smmu->version = ARM_SMMU_V2;
1933                 smmu->model = ARM_MMU500;
1934                 break;
1935         default:
1936                 ret = -ENODEV;
1937         }
1938 
1939         return ret;
1940 }
1941 
1942 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1943                                       struct arm_smmu_device *smmu)
1944 {
1945         struct device *dev = smmu->dev;
1946         struct acpi_iort_node *node =
1947                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1948         struct acpi_iort_smmu *iort_smmu;
1949         int ret;
1950 
1951         /* Retrieve SMMU1/2 specific data */
1952         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1953 
1954         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1955         if (ret < 0)
1956                 return ret;
1957 
1958         /* Ignore the configuration access interrupt */
1959         smmu->num_global_irqs = 1;
1960 
1961         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1962                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1963 
1964         return 0;
1965 }
1966 #else
1967 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1968                                              struct arm_smmu_device *smmu)
1969 {
1970         return -ENODEV;
1971 }
1972 #endif
1973 
1974 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1975                                     struct arm_smmu_device *smmu)
1976 {
1977         const struct arm_smmu_match_data *data;
1978         struct device *dev = &pdev->dev;
1979         bool legacy_binding;
1980 
1981         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1982                                  &smmu->num_global_irqs)) {
1983                 dev_err(dev, "missing #global-interrupts property\n");
1984                 return -ENODEV;
1985         }
1986 
1987         data = of_device_get_match_data(dev);
1988         smmu->version = data->version;
1989         smmu->model = data->model;
1990 
1991         parse_driver_options(smmu);
1992 
1993         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1994         if (legacy_binding && !using_generic_binding) {
1995                 if (!using_legacy_binding)
1996                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1997                 using_legacy_binding = true;
1998         } else if (!legacy_binding && !using_legacy_binding) {
1999                 using_generic_binding = true;
2000         } else {
2001                 dev_err(dev, "not probing due to mismatched DT properties\n");
2002                 return -ENODEV;
2003         }
2004 
2005         if (of_dma_is_coherent(dev->of_node))
2006                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2007 
2008         return 0;
2009 }
2010 
2011 static int arm_smmu_device_probe(struct platform_device *pdev)
2012 {
2013         struct resource *res;
2014         struct arm_smmu_device *smmu;
2015         struct device *dev = &pdev->dev;
2016         int num_irqs, i, err;
2017 
2018         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2019         if (!smmu) {
2020                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2021                 return -ENOMEM;
2022         }
2023         smmu->dev = dev;
2024 
2025         if (dev->of_node)
2026                 err = arm_smmu_device_dt_probe(pdev, smmu);
2027         else
2028                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2029 
2030         if (err)
2031                 return err;
2032 
2033         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2034         smmu->base = devm_ioremap_resource(dev, res);
2035         if (IS_ERR(smmu->base))
2036                 return PTR_ERR(smmu->base);
2037         smmu->size = resource_size(res);
2038 
2039         num_irqs = 0;
2040         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2041                 num_irqs++;
2042                 if (num_irqs > smmu->num_global_irqs)
2043                         smmu->num_context_irqs++;
2044         }
2045 
2046         if (!smmu->num_context_irqs) {
2047                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2048                         num_irqs, smmu->num_global_irqs + 1);
2049                 return -ENODEV;
2050         }
2051 
2052         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2053                                   GFP_KERNEL);
2054         if (!smmu->irqs) {
2055                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2056                 return -ENOMEM;
2057         }
2058 
2059         for (i = 0; i < num_irqs; ++i) {
2060                 int irq = platform_get_irq(pdev, i);
2061 
2062                 if (irq < 0) {
2063                         dev_err(dev, "failed to get irq index %d\n", i);
2064                         return -ENODEV;
2065                 }
2066                 smmu->irqs[i] = irq;
2067         }
2068 
2069         err = arm_smmu_device_cfg_probe(smmu);
2070         if (err)
2071                 return err;
2072 
2073         if (smmu->version == ARM_SMMU_V2 &&
2074             smmu->num_context_banks != smmu->num_context_irqs) {
2075                 dev_err(dev,
2076                         "found only %d context interrupt(s) but %d required\n",
2077                         smmu->num_context_irqs, smmu->num_context_banks);
2078                 return -ENODEV;
2079         }
2080 
2081         for (i = 0; i < smmu->num_global_irqs; ++i) {
2082                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2083                                        arm_smmu_global_fault,
2084                                        IRQF_SHARED,
2085                                        "arm-smmu global fault",
2086                                        smmu);
2087                 if (err) {
2088                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2089                                 i, smmu->irqs[i]);
2090                         return err;
2091                 }
2092         }
2093 
2094         iommu_register_instance(dev->fwnode, &arm_smmu_ops);
2095         platform_set_drvdata(pdev, smmu);
2096         arm_smmu_device_reset(smmu);
2097 
2098         /* Oh, for a proper bus abstraction */
2099         if (!iommu_present(&platform_bus_type))
2100                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2101 #ifdef CONFIG_ARM_AMBA
2102         if (!iommu_present(&amba_bustype))
2103                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2104 #endif
2105 #ifdef CONFIG_PCI
2106         if (!iommu_present(&pci_bus_type)) {
2107                 pci_request_acs();
2108                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2109         }
2110 #endif
2111         return 0;
2112 }
2113 
2114 static int arm_smmu_device_remove(struct platform_device *pdev)
2115 {
2116         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2117 
2118         if (!smmu)
2119                 return -ENODEV;
2120 
2121         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2122                 dev_err(&pdev->dev, "removing device with active domains!\n");
2123 
2124         /* Turn the thing off */
2125         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2126         return 0;
2127 }
2128 
2129 static struct platform_driver arm_smmu_driver = {
2130         .driver = {
2131                 .name           = "arm-smmu",
2132                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2133         },
2134         .probe  = arm_smmu_device_probe,
2135         .remove = arm_smmu_device_remove,
2136 };
2137 
2138 static int __init arm_smmu_init(void)
2139 {
2140         static bool registered;
2141         int ret = 0;
2142 
2143         if (!registered) {
2144                 ret = platform_driver_register(&arm_smmu_driver);
2145                 registered = !ret;
2146         }
2147         return ret;
2148 }
2149 
2150 static void __exit arm_smmu_exit(void)
2151 {
2152         return platform_driver_unregister(&arm_smmu_driver);
2153 }
2154 
2155 subsys_initcall(arm_smmu_init);
2156 module_exit(arm_smmu_exit);
2157 
2158 static int __init arm_smmu_of_init(struct device_node *np)
2159 {
2160         int ret = arm_smmu_init();
2161 
2162         if (ret)
2163                 return ret;
2164 
2165         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2166                 return -ENODEV;
2167 
2168         return 0;
2169 }
2170 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2171 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2172 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2173 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2174 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2175 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2176 
2177 #ifdef CONFIG_ACPI
2178 static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2179 {
2180         if (iort_node_match(ACPI_IORT_NODE_SMMU))
2181                 return arm_smmu_init();
2182 
2183         return 0;
2184 }
2185 IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2186 #endif
2187 
2188 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2189 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2190 MODULE_LICENSE("GPL v2");
2191 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us