Version:  2.0.40 2.2.26 2.4.37 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9

Linux/drivers/iommu/arm-smmu.c

  1 /*
  2  * IOMMU API for ARM architected SMMU implementations.
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  *
  8  * This program is distributed in the hope that it will be useful,
  9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 11  * GNU General Public License for more details.
 12  *
 13  * You should have received a copy of the GNU General Public License
 14  * along with this program; if not, write to the Free Software
 15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 16  *
 17  * Copyright (C) 2013 ARM Limited
 18  *
 19  * Author: Will Deacon <will.deacon@arm.com>
 20  *
 21  * This driver currently supports:
 22  *      - SMMUv1 and v2 implementations
 23  *      - Stream-matching and stream-indexing
 24  *      - v7/v8 long-descriptor format
 25  *      - Non-secure access to the SMMU
 26  *      - Context fault reporting
 27  */
 28 
 29 #define pr_fmt(fmt) "arm-smmu: " fmt
 30 
 31 #include <linux/atomic.h>
 32 #include <linux/delay.h>
 33 #include <linux/dma-iommu.h>
 34 #include <linux/dma-mapping.h>
 35 #include <linux/err.h>
 36 #include <linux/interrupt.h>
 37 #include <linux/io.h>
 38 #include <linux/io-64-nonatomic-hi-lo.h>
 39 #include <linux/iommu.h>
 40 #include <linux/iopoll.h>
 41 #include <linux/module.h>
 42 #include <linux/of.h>
 43 #include <linux/of_address.h>
 44 #include <linux/of_device.h>
 45 #include <linux/of_iommu.h>
 46 #include <linux/pci.h>
 47 #include <linux/platform_device.h>
 48 #include <linux/slab.h>
 49 #include <linux/spinlock.h>
 50 
 51 #include <linux/amba/bus.h>
 52 
 53 #include "io-pgtable.h"
 54 
 55 /* Maximum number of context banks per SMMU */
 56 #define ARM_SMMU_MAX_CBS                128
 57 
 58 /* SMMU global address space */
 59 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
 60 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
 61 
 62 /*
 63  * SMMU global address space with conditional offset to access secure
 64  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
 65  * nsGFSYNR0: 0x450)
 66  */
 67 #define ARM_SMMU_GR0_NS(smmu)                                           \
 68         ((smmu)->base +                                                 \
 69                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
 70                         ? 0x400 : 0))
 71 
 72 /*
 73  * Some 64-bit registers only make sense to write atomically, but in such
 74  * cases all the data relevant to AArch32 formats lies within the lower word,
 75  * therefore this actually makes more sense than it might first appear.
 76  */
 77 #ifdef CONFIG_64BIT
 78 #define smmu_write_atomic_lq            writeq_relaxed
 79 #else
 80 #define smmu_write_atomic_lq            writel_relaxed
 81 #endif
 82 
 83 /* Configuration registers */
 84 #define ARM_SMMU_GR0_sCR0               0x0
 85 #define sCR0_CLIENTPD                   (1 << 0)
 86 #define sCR0_GFRE                       (1 << 1)
 87 #define sCR0_GFIE                       (1 << 2)
 88 #define sCR0_GCFGFRE                    (1 << 4)
 89 #define sCR0_GCFGFIE                    (1 << 5)
 90 #define sCR0_USFCFG                     (1 << 10)
 91 #define sCR0_VMIDPNE                    (1 << 11)
 92 #define sCR0_PTM                        (1 << 12)
 93 #define sCR0_FB                         (1 << 13)
 94 #define sCR0_VMID16EN                   (1 << 31)
 95 #define sCR0_BSU_SHIFT                  14
 96 #define sCR0_BSU_MASK                   0x3
 97 
 98 /* Auxiliary Configuration register */
 99 #define ARM_SMMU_GR0_sACR               0x10
100 
101 /* Identification registers */
102 #define ARM_SMMU_GR0_ID0                0x20
103 #define ARM_SMMU_GR0_ID1                0x24
104 #define ARM_SMMU_GR0_ID2                0x28
105 #define ARM_SMMU_GR0_ID3                0x2c
106 #define ARM_SMMU_GR0_ID4                0x30
107 #define ARM_SMMU_GR0_ID5                0x34
108 #define ARM_SMMU_GR0_ID6                0x38
109 #define ARM_SMMU_GR0_ID7                0x3c
110 #define ARM_SMMU_GR0_sGFSR              0x48
111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
114 
115 #define ID0_S1TS                        (1 << 30)
116 #define ID0_S2TS                        (1 << 29)
117 #define ID0_NTS                         (1 << 28)
118 #define ID0_SMS                         (1 << 27)
119 #define ID0_ATOSNS                      (1 << 26)
120 #define ID0_PTFS_NO_AARCH32             (1 << 25)
121 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
122 #define ID0_CTTW                        (1 << 14)
123 #define ID0_NUMIRPT_SHIFT               16
124 #define ID0_NUMIRPT_MASK                0xff
125 #define ID0_NUMSIDB_SHIFT               9
126 #define ID0_NUMSIDB_MASK                0xf
127 #define ID0_NUMSMRG_SHIFT               0
128 #define ID0_NUMSMRG_MASK                0xff
129 
130 #define ID1_PAGESIZE                    (1 << 31)
131 #define ID1_NUMPAGENDXB_SHIFT           28
132 #define ID1_NUMPAGENDXB_MASK            7
133 #define ID1_NUMS2CB_SHIFT               16
134 #define ID1_NUMS2CB_MASK                0xff
135 #define ID1_NUMCB_SHIFT                 0
136 #define ID1_NUMCB_MASK                  0xff
137 
138 #define ID2_OAS_SHIFT                   4
139 #define ID2_OAS_MASK                    0xf
140 #define ID2_IAS_SHIFT                   0
141 #define ID2_IAS_MASK                    0xf
142 #define ID2_UBS_SHIFT                   8
143 #define ID2_UBS_MASK                    0xf
144 #define ID2_PTFS_4K                     (1 << 12)
145 #define ID2_PTFS_16K                    (1 << 13)
146 #define ID2_PTFS_64K                    (1 << 14)
147 #define ID2_VMID16                      (1 << 15)
148 
149 #define ID7_MAJOR_SHIFT                 4
150 #define ID7_MAJOR_MASK                  0xf
151 
152 /* Global TLB invalidation */
153 #define ARM_SMMU_GR0_TLBIVMID           0x64
154 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
155 #define ARM_SMMU_GR0_TLBIALLH           0x6c
156 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
157 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
158 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
159 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
160 
161 /* Stream mapping registers */
162 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
163 #define SMR_VALID                       (1 << 31)
164 #define SMR_MASK_SHIFT                  16
165 #define SMR_ID_SHIFT                    0
166 
167 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
168 #define S2CR_CBNDX_SHIFT                0
169 #define S2CR_CBNDX_MASK                 0xff
170 #define S2CR_TYPE_SHIFT                 16
171 #define S2CR_TYPE_MASK                  0x3
172 enum arm_smmu_s2cr_type {
173         S2CR_TYPE_TRANS,
174         S2CR_TYPE_BYPASS,
175         S2CR_TYPE_FAULT,
176 };
177 
178 #define S2CR_PRIVCFG_SHIFT              24
179 #define S2CR_PRIVCFG_MASK               0x3
180 enum arm_smmu_s2cr_privcfg {
181         S2CR_PRIVCFG_DEFAULT,
182         S2CR_PRIVCFG_DIPAN,
183         S2CR_PRIVCFG_UNPRIV,
184         S2CR_PRIVCFG_PRIV,
185 };
186 
187 /* Context bank attribute registers */
188 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
189 #define CBAR_VMID_SHIFT                 0
190 #define CBAR_VMID_MASK                  0xff
191 #define CBAR_S1_BPSHCFG_SHIFT           8
192 #define CBAR_S1_BPSHCFG_MASK            3
193 #define CBAR_S1_BPSHCFG_NSH             3
194 #define CBAR_S1_MEMATTR_SHIFT           12
195 #define CBAR_S1_MEMATTR_MASK            0xf
196 #define CBAR_S1_MEMATTR_WB              0xf
197 #define CBAR_TYPE_SHIFT                 16
198 #define CBAR_TYPE_MASK                  0x3
199 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
201 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
202 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
203 #define CBAR_IRPTNDX_SHIFT              24
204 #define CBAR_IRPTNDX_MASK               0xff
205 
206 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
207 #define CBA2R_RW64_32BIT                (0 << 0)
208 #define CBA2R_RW64_64BIT                (1 << 0)
209 #define CBA2R_VMID_SHIFT                16
210 #define CBA2R_VMID_MASK                 0xffff
211 
212 /* Translation context bank */
213 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
214 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
215 
216 #define ARM_SMMU_CB_SCTLR               0x0
217 #define ARM_SMMU_CB_ACTLR               0x4
218 #define ARM_SMMU_CB_RESUME              0x8
219 #define ARM_SMMU_CB_TTBCR2              0x10
220 #define ARM_SMMU_CB_TTBR0               0x20
221 #define ARM_SMMU_CB_TTBR1               0x28
222 #define ARM_SMMU_CB_TTBCR               0x30
223 #define ARM_SMMU_CB_CONTEXTIDR          0x34
224 #define ARM_SMMU_CB_S1_MAIR0            0x38
225 #define ARM_SMMU_CB_S1_MAIR1            0x3c
226 #define ARM_SMMU_CB_PAR                 0x50
227 #define ARM_SMMU_CB_FSR                 0x58
228 #define ARM_SMMU_CB_FAR                 0x60
229 #define ARM_SMMU_CB_FSYNR0              0x68
230 #define ARM_SMMU_CB_S1_TLBIVA           0x600
231 #define ARM_SMMU_CB_S1_TLBIASID         0x610
232 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
233 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
234 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
235 #define ARM_SMMU_CB_ATS1PR              0x800
236 #define ARM_SMMU_CB_ATSR                0x8f0
237 
238 #define SCTLR_S1_ASIDPNE                (1 << 12)
239 #define SCTLR_CFCFG                     (1 << 7)
240 #define SCTLR_CFIE                      (1 << 6)
241 #define SCTLR_CFRE                      (1 << 5)
242 #define SCTLR_E                         (1 << 4)
243 #define SCTLR_AFE                       (1 << 2)
244 #define SCTLR_TRE                       (1 << 1)
245 #define SCTLR_M                         (1 << 0)
246 
247 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
248 
249 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
250 
251 #define CB_PAR_F                        (1 << 0)
252 
253 #define ATSR_ACTIVE                     (1 << 0)
254 
255 #define RESUME_RETRY                    (0 << 0)
256 #define RESUME_TERMINATE                (1 << 0)
257 
258 #define TTBCR2_SEP_SHIFT                15
259 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
260 
261 #define TTBRn_ASID_SHIFT                48
262 
263 #define FSR_MULTI                       (1 << 31)
264 #define FSR_SS                          (1 << 30)
265 #define FSR_UUT                         (1 << 8)
266 #define FSR_ASF                         (1 << 7)
267 #define FSR_TLBLKF                      (1 << 6)
268 #define FSR_TLBMCF                      (1 << 5)
269 #define FSR_EF                          (1 << 4)
270 #define FSR_PF                          (1 << 3)
271 #define FSR_AFF                         (1 << 2)
272 #define FSR_TF                          (1 << 1)
273 
274 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
275                                          FSR_TLBMCF | FSR_TLBLKF)
276 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
277                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
278 
279 #define FSYNR0_WNR                      (1 << 4)
280 
281 static int force_stage;
282 module_param(force_stage, int, S_IRUGO);
283 MODULE_PARM_DESC(force_stage,
284         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
285 static bool disable_bypass;
286 module_param(disable_bypass, bool, S_IRUGO);
287 MODULE_PARM_DESC(disable_bypass,
288         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
289 
290 enum arm_smmu_arch_version {
291         ARM_SMMU_V1,
292         ARM_SMMU_V1_64K,
293         ARM_SMMU_V2,
294 };
295 
296 enum arm_smmu_implementation {
297         GENERIC_SMMU,
298         ARM_MMU500,
299         CAVIUM_SMMUV2,
300 };
301 
302 struct arm_smmu_s2cr {
303         struct iommu_group              *group;
304         int                             count;
305         enum arm_smmu_s2cr_type         type;
306         enum arm_smmu_s2cr_privcfg      privcfg;
307         u8                              cbndx;
308 };
309 
310 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
311         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
312 }
313 
314 struct arm_smmu_smr {
315         u16                             mask;
316         u16                             id;
317         bool                            valid;
318 };
319 
320 struct arm_smmu_master_cfg {
321         struct arm_smmu_device          *smmu;
322         s16                             smendx[];
323 };
324 #define INVALID_SMENDX                  -1
325 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
326 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
327 #define fwspec_smendx(fw, i) \
328         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
329 #define for_each_cfg_sme(fw, i, idx) \
330         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
331 
332 struct arm_smmu_device {
333         struct device                   *dev;
334 
335         void __iomem                    *base;
336         unsigned long                   size;
337         unsigned long                   pgshift;
338 
339 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
340 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
341 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
342 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
343 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
344 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
345 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
346 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
347 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
348 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
349 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
350 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
351         u32                             features;
352 
353 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
354         u32                             options;
355         enum arm_smmu_arch_version      version;
356         enum arm_smmu_implementation    model;
357 
358         u32                             num_context_banks;
359         u32                             num_s2_context_banks;
360         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
361         atomic_t                        irptndx;
362 
363         u32                             num_mapping_groups;
364         u16                             streamid_mask;
365         u16                             smr_mask_mask;
366         struct arm_smmu_smr             *smrs;
367         struct arm_smmu_s2cr            *s2crs;
368         struct mutex                    stream_map_mutex;
369 
370         unsigned long                   va_size;
371         unsigned long                   ipa_size;
372         unsigned long                   pa_size;
373         unsigned long                   pgsize_bitmap;
374 
375         u32                             num_global_irqs;
376         u32                             num_context_irqs;
377         unsigned int                    *irqs;
378 
379         u32                             cavium_id_base; /* Specific to Cavium */
380 };
381 
382 enum arm_smmu_context_fmt {
383         ARM_SMMU_CTX_FMT_NONE,
384         ARM_SMMU_CTX_FMT_AARCH64,
385         ARM_SMMU_CTX_FMT_AARCH32_L,
386         ARM_SMMU_CTX_FMT_AARCH32_S,
387 };
388 
389 struct arm_smmu_cfg {
390         u8                              cbndx;
391         u8                              irptndx;
392         u32                             cbar;
393         enum arm_smmu_context_fmt       fmt;
394 };
395 #define INVALID_IRPTNDX                 0xff
396 
397 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
398 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
399 
400 enum arm_smmu_domain_stage {
401         ARM_SMMU_DOMAIN_S1 = 0,
402         ARM_SMMU_DOMAIN_S2,
403         ARM_SMMU_DOMAIN_NESTED,
404 };
405 
406 struct arm_smmu_domain {
407         struct arm_smmu_device          *smmu;
408         struct io_pgtable_ops           *pgtbl_ops;
409         spinlock_t                      pgtbl_lock;
410         struct arm_smmu_cfg             cfg;
411         enum arm_smmu_domain_stage      stage;
412         struct mutex                    init_mutex; /* Protects smmu pointer */
413         struct iommu_domain             domain;
414 };
415 
416 struct arm_smmu_option_prop {
417         u32 opt;
418         const char *prop;
419 };
420 
421 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
422 
423 static bool using_legacy_binding, using_generic_binding;
424 
425 static struct arm_smmu_option_prop arm_smmu_options[] = {
426         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
427         { 0, NULL},
428 };
429 
430 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
431 {
432         return container_of(dom, struct arm_smmu_domain, domain);
433 }
434 
435 static void parse_driver_options(struct arm_smmu_device *smmu)
436 {
437         int i = 0;
438 
439         do {
440                 if (of_property_read_bool(smmu->dev->of_node,
441                                                 arm_smmu_options[i].prop)) {
442                         smmu->options |= arm_smmu_options[i].opt;
443                         dev_notice(smmu->dev, "option %s\n",
444                                 arm_smmu_options[i].prop);
445                 }
446         } while (arm_smmu_options[++i].opt);
447 }
448 
449 static struct device_node *dev_get_dev_node(struct device *dev)
450 {
451         if (dev_is_pci(dev)) {
452                 struct pci_bus *bus = to_pci_dev(dev)->bus;
453 
454                 while (!pci_is_root_bus(bus))
455                         bus = bus->parent;
456                 return of_node_get(bus->bridge->parent->of_node);
457         }
458 
459         return of_node_get(dev->of_node);
460 }
461 
462 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
463 {
464         *((__be32 *)data) = cpu_to_be32(alias);
465         return 0; /* Continue walking */
466 }
467 
468 static int __find_legacy_master_phandle(struct device *dev, void *data)
469 {
470         struct of_phandle_iterator *it = *(void **)data;
471         struct device_node *np = it->node;
472         int err;
473 
474         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
475                             "#stream-id-cells", 0)
476                 if (it->node == np) {
477                         *(void **)data = dev;
478                         return 1;
479                 }
480         it->node = np;
481         return err == -ENOENT ? 0 : err;
482 }
483 
484 static struct platform_driver arm_smmu_driver;
485 static struct iommu_ops arm_smmu_ops;
486 
487 static int arm_smmu_register_legacy_master(struct device *dev,
488                                            struct arm_smmu_device **smmu)
489 {
490         struct device *smmu_dev;
491         struct device_node *np;
492         struct of_phandle_iterator it;
493         void *data = &it;
494         u32 *sids;
495         __be32 pci_sid;
496         int err;
497 
498         np = dev_get_dev_node(dev);
499         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
500                 of_node_put(np);
501                 return -ENODEV;
502         }
503 
504         it.node = np;
505         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
506                                      __find_legacy_master_phandle);
507         smmu_dev = data;
508         of_node_put(np);
509         if (err == 0)
510                 return -ENODEV;
511         if (err < 0)
512                 return err;
513 
514         if (dev_is_pci(dev)) {
515                 /* "mmu-masters" assumes Stream ID == Requester ID */
516                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
517                                        &pci_sid);
518                 it.cur = &pci_sid;
519                 it.cur_count = 1;
520         }
521 
522         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
523                                 &arm_smmu_ops);
524         if (err)
525                 return err;
526 
527         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
528         if (!sids)
529                 return -ENOMEM;
530 
531         *smmu = dev_get_drvdata(smmu_dev);
532         of_phandle_iterator_args(&it, sids, it.cur_count);
533         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
534         kfree(sids);
535         return err;
536 }
537 
538 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
539 {
540         int idx;
541 
542         do {
543                 idx = find_next_zero_bit(map, end, start);
544                 if (idx == end)
545                         return -ENOSPC;
546         } while (test_and_set_bit(idx, map));
547 
548         return idx;
549 }
550 
551 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
552 {
553         clear_bit(idx, map);
554 }
555 
556 /* Wait for any pending TLB invalidations to complete */
557 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
558 {
559         int count = 0;
560         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
561 
562         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
563         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
564                & sTLBGSTATUS_GSACTIVE) {
565                 cpu_relax();
566                 if (++count == TLB_LOOP_TIMEOUT) {
567                         dev_err_ratelimited(smmu->dev,
568                         "TLB sync timed out -- SMMU may be deadlocked\n");
569                         return;
570                 }
571                 udelay(1);
572         }
573 }
574 
575 static void arm_smmu_tlb_sync(void *cookie)
576 {
577         struct arm_smmu_domain *smmu_domain = cookie;
578         __arm_smmu_tlb_sync(smmu_domain->smmu);
579 }
580 
581 static void arm_smmu_tlb_inv_context(void *cookie)
582 {
583         struct arm_smmu_domain *smmu_domain = cookie;
584         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
585         struct arm_smmu_device *smmu = smmu_domain->smmu;
586         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
587         void __iomem *base;
588 
589         if (stage1) {
590                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
591                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
592                                base + ARM_SMMU_CB_S1_TLBIASID);
593         } else {
594                 base = ARM_SMMU_GR0(smmu);
595                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
596                                base + ARM_SMMU_GR0_TLBIVMID);
597         }
598 
599         __arm_smmu_tlb_sync(smmu);
600 }
601 
602 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
603                                           size_t granule, bool leaf, void *cookie)
604 {
605         struct arm_smmu_domain *smmu_domain = cookie;
606         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
607         struct arm_smmu_device *smmu = smmu_domain->smmu;
608         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
609         void __iomem *reg;
610 
611         if (stage1) {
612                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
613                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
614 
615                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
616                         iova &= ~12UL;
617                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
618                         do {
619                                 writel_relaxed(iova, reg);
620                                 iova += granule;
621                         } while (size -= granule);
622                 } else {
623                         iova >>= 12;
624                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
625                         do {
626                                 writeq_relaxed(iova, reg);
627                                 iova += granule >> 12;
628                         } while (size -= granule);
629                 }
630         } else if (smmu->version == ARM_SMMU_V2) {
631                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
632                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
633                               ARM_SMMU_CB_S2_TLBIIPAS2;
634                 iova >>= 12;
635                 do {
636                         smmu_write_atomic_lq(iova, reg);
637                         iova += granule >> 12;
638                 } while (size -= granule);
639         } else {
640                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
641                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
642         }
643 }
644 
645 static struct iommu_gather_ops arm_smmu_gather_ops = {
646         .tlb_flush_all  = arm_smmu_tlb_inv_context,
647         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
648         .tlb_sync       = arm_smmu_tlb_sync,
649 };
650 
651 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
652 {
653         u32 fsr, fsynr;
654         unsigned long iova;
655         struct iommu_domain *domain = dev;
656         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
657         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
658         struct arm_smmu_device *smmu = smmu_domain->smmu;
659         void __iomem *cb_base;
660 
661         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
662         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
663 
664         if (!(fsr & FSR_FAULT))
665                 return IRQ_NONE;
666 
667         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
668         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
669 
670         dev_err_ratelimited(smmu->dev,
671         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
672                             fsr, iova, fsynr, cfg->cbndx);
673 
674         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
675         return IRQ_HANDLED;
676 }
677 
678 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
679 {
680         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
681         struct arm_smmu_device *smmu = dev;
682         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
683 
684         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
685         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
686         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
687         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
688 
689         if (!gfsr)
690                 return IRQ_NONE;
691 
692         dev_err_ratelimited(smmu->dev,
693                 "Unexpected global fault, this could be serious\n");
694         dev_err_ratelimited(smmu->dev,
695                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
696                 gfsr, gfsynr0, gfsynr1, gfsynr2);
697 
698         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
699         return IRQ_HANDLED;
700 }
701 
702 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
703                                        struct io_pgtable_cfg *pgtbl_cfg)
704 {
705         u32 reg, reg2;
706         u64 reg64;
707         bool stage1;
708         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
709         struct arm_smmu_device *smmu = smmu_domain->smmu;
710         void __iomem *cb_base, *gr1_base;
711 
712         gr1_base = ARM_SMMU_GR1(smmu);
713         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
714         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
715 
716         if (smmu->version > ARM_SMMU_V1) {
717                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
718                         reg = CBA2R_RW64_64BIT;
719                 else
720                         reg = CBA2R_RW64_32BIT;
721                 /* 16-bit VMIDs live in CBA2R */
722                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
723                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
724 
725                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
726         }
727 
728         /* CBAR */
729         reg = cfg->cbar;
730         if (smmu->version < ARM_SMMU_V2)
731                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
732 
733         /*
734          * Use the weakest shareability/memory types, so they are
735          * overridden by the ttbcr/pte.
736          */
737         if (stage1) {
738                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
739                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
740         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
741                 /* 8-bit VMIDs live in CBAR */
742                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
743         }
744         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
745 
746         /* TTBRs */
747         if (stage1) {
748                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
749 
750                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
751                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
752                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
753                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
754                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
755                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
756                 } else {
757                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
758                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
759                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
760                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
761                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
762                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
763                 }
764         } else {
765                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
766                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
767         }
768 
769         /* TTBCR */
770         if (stage1) {
771                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
772                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
773                         reg2 = 0;
774                 } else {
775                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
776                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
777                         reg2 |= TTBCR2_SEP_UPSTREAM;
778                 }
779                 if (smmu->version > ARM_SMMU_V1)
780                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
781         } else {
782                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
783         }
784         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
785 
786         /* MAIRs (stage-1 only) */
787         if (stage1) {
788                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
789                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
790                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
791                 } else {
792                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
793                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
794                 }
795                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
796                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
797         }
798 
799         /* SCTLR */
800         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
801         if (stage1)
802                 reg |= SCTLR_S1_ASIDPNE;
803 #ifdef __BIG_ENDIAN
804         reg |= SCTLR_E;
805 #endif
806         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
807 }
808 
809 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
810                                         struct arm_smmu_device *smmu)
811 {
812         int irq, start, ret = 0;
813         unsigned long ias, oas;
814         struct io_pgtable_ops *pgtbl_ops;
815         struct io_pgtable_cfg pgtbl_cfg;
816         enum io_pgtable_fmt fmt;
817         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
818         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
819 
820         mutex_lock(&smmu_domain->init_mutex);
821         if (smmu_domain->smmu)
822                 goto out_unlock;
823 
824         /*
825          * Mapping the requested stage onto what we support is surprisingly
826          * complicated, mainly because the spec allows S1+S2 SMMUs without
827          * support for nested translation. That means we end up with the
828          * following table:
829          *
830          * Requested        Supported        Actual
831          *     S1               N              S1
832          *     S1             S1+S2            S1
833          *     S1               S2             S2
834          *     S1               S1             S1
835          *     N                N              N
836          *     N              S1+S2            S2
837          *     N                S2             S2
838          *     N                S1             S1
839          *
840          * Note that you can't actually request stage-2 mappings.
841          */
842         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
843                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
844         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
845                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
846 
847         /*
848          * Choosing a suitable context format is even more fiddly. Until we
849          * grow some way for the caller to express a preference, and/or move
850          * the decision into the io-pgtable code where it arguably belongs,
851          * just aim for the closest thing to the rest of the system, and hope
852          * that the hardware isn't esoteric enough that we can't assume AArch64
853          * support to be a superset of AArch32 support...
854          */
855         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
856                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
857         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
858             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
859             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
860             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
861                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
862         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
863             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
864                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
865                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
866                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
867 
868         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
869                 ret = -EINVAL;
870                 goto out_unlock;
871         }
872 
873         switch (smmu_domain->stage) {
874         case ARM_SMMU_DOMAIN_S1:
875                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
876                 start = smmu->num_s2_context_banks;
877                 ias = smmu->va_size;
878                 oas = smmu->ipa_size;
879                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
880                         fmt = ARM_64_LPAE_S1;
881                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
882                         fmt = ARM_32_LPAE_S1;
883                         ias = min(ias, 32UL);
884                         oas = min(oas, 40UL);
885                 } else {
886                         fmt = ARM_V7S;
887                         ias = min(ias, 32UL);
888                         oas = min(oas, 32UL);
889                 }
890                 break;
891         case ARM_SMMU_DOMAIN_NESTED:
892                 /*
893                  * We will likely want to change this if/when KVM gets
894                  * involved.
895                  */
896         case ARM_SMMU_DOMAIN_S2:
897                 cfg->cbar = CBAR_TYPE_S2_TRANS;
898                 start = 0;
899                 ias = smmu->ipa_size;
900                 oas = smmu->pa_size;
901                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
902                         fmt = ARM_64_LPAE_S2;
903                 } else {
904                         fmt = ARM_32_LPAE_S2;
905                         ias = min(ias, 40UL);
906                         oas = min(oas, 40UL);
907                 }
908                 break;
909         default:
910                 ret = -EINVAL;
911                 goto out_unlock;
912         }
913 
914         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
915                                       smmu->num_context_banks);
916         if (ret < 0)
917                 goto out_unlock;
918 
919         cfg->cbndx = ret;
920         if (smmu->version < ARM_SMMU_V2) {
921                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
922                 cfg->irptndx %= smmu->num_context_irqs;
923         } else {
924                 cfg->irptndx = cfg->cbndx;
925         }
926 
927         pgtbl_cfg = (struct io_pgtable_cfg) {
928                 .pgsize_bitmap  = smmu->pgsize_bitmap,
929                 .ias            = ias,
930                 .oas            = oas,
931                 .tlb            = &arm_smmu_gather_ops,
932                 .iommu_dev      = smmu->dev,
933         };
934 
935         smmu_domain->smmu = smmu;
936         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
937         if (!pgtbl_ops) {
938                 ret = -ENOMEM;
939                 goto out_clear_smmu;
940         }
941 
942         /* Update the domain's page sizes to reflect the page table format */
943         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
944         domain->geometry.aperture_end = (1UL << ias) - 1;
945         domain->geometry.force_aperture = true;
946 
947         /* Initialise the context bank with our page table cfg */
948         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
949 
950         /*
951          * Request context fault interrupt. Do this last to avoid the
952          * handler seeing a half-initialised domain state.
953          */
954         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
955         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
956                                IRQF_SHARED, "arm-smmu-context-fault", domain);
957         if (ret < 0) {
958                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
959                         cfg->irptndx, irq);
960                 cfg->irptndx = INVALID_IRPTNDX;
961         }
962 
963         mutex_unlock(&smmu_domain->init_mutex);
964 
965         /* Publish page table ops for map/unmap */
966         smmu_domain->pgtbl_ops = pgtbl_ops;
967         return 0;
968 
969 out_clear_smmu:
970         smmu_domain->smmu = NULL;
971 out_unlock:
972         mutex_unlock(&smmu_domain->init_mutex);
973         return ret;
974 }
975 
976 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
977 {
978         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
979         struct arm_smmu_device *smmu = smmu_domain->smmu;
980         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
981         void __iomem *cb_base;
982         int irq;
983 
984         if (!smmu)
985                 return;
986 
987         /*
988          * Disable the context bank and free the page tables before freeing
989          * it.
990          */
991         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
992         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
993 
994         if (cfg->irptndx != INVALID_IRPTNDX) {
995                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
996                 devm_free_irq(smmu->dev, irq, domain);
997         }
998 
999         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1000         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1001 }
1002 
1003 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1004 {
1005         struct arm_smmu_domain *smmu_domain;
1006 
1007         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1008                 return NULL;
1009         /*
1010          * Allocate the domain and initialise some of its data structures.
1011          * We can't really do anything meaningful until we've added a
1012          * master.
1013          */
1014         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1015         if (!smmu_domain)
1016                 return NULL;
1017 
1018         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1019             iommu_get_dma_cookie(&smmu_domain->domain))) {
1020                 kfree(smmu_domain);
1021                 return NULL;
1022         }
1023 
1024         mutex_init(&smmu_domain->init_mutex);
1025         spin_lock_init(&smmu_domain->pgtbl_lock);
1026 
1027         return &smmu_domain->domain;
1028 }
1029 
1030 static void arm_smmu_domain_free(struct iommu_domain *domain)
1031 {
1032         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1033 
1034         /*
1035          * Free the domain resources. We assume that all devices have
1036          * already been detached.
1037          */
1038         iommu_put_dma_cookie(domain);
1039         arm_smmu_destroy_domain_context(domain);
1040         kfree(smmu_domain);
1041 }
1042 
1043 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1044 {
1045         struct arm_smmu_smr *smr = smmu->smrs + idx;
1046         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1047 
1048         if (smr->valid)
1049                 reg |= SMR_VALID;
1050         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1051 }
1052 
1053 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1054 {
1055         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1056         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1057                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1058                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1059 
1060         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1061 }
1062 
1063 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1064 {
1065         arm_smmu_write_s2cr(smmu, idx);
1066         if (smmu->smrs)
1067                 arm_smmu_write_smr(smmu, idx);
1068 }
1069 
1070 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1071 {
1072         struct arm_smmu_smr *smrs = smmu->smrs;
1073         int i, free_idx = -ENOSPC;
1074 
1075         /* Stream indexing is blissfully easy */
1076         if (!smrs)
1077                 return id;
1078 
1079         /* Validating SMRs is... less so */
1080         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1081                 if (!smrs[i].valid) {
1082                         /*
1083                          * Note the first free entry we come across, which
1084                          * we'll claim in the end if nothing else matches.
1085                          */
1086                         if (free_idx < 0)
1087                                 free_idx = i;
1088                         continue;
1089                 }
1090                 /*
1091                  * If the new entry is _entirely_ matched by an existing entry,
1092                  * then reuse that, with the guarantee that there also cannot
1093                  * be any subsequent conflicting entries. In normal use we'd
1094                  * expect simply identical entries for this case, but there's
1095                  * no harm in accommodating the generalisation.
1096                  */
1097                 if ((mask & smrs[i].mask) == mask &&
1098                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1099                         return i;
1100                 /*
1101                  * If the new entry has any other overlap with an existing one,
1102                  * though, then there always exists at least one stream ID
1103                  * which would cause a conflict, and we can't allow that risk.
1104                  */
1105                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1106                         return -EINVAL;
1107         }
1108 
1109         return free_idx;
1110 }
1111 
1112 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1113 {
1114         if (--smmu->s2crs[idx].count)
1115                 return false;
1116 
1117         smmu->s2crs[idx] = s2cr_init_val;
1118         if (smmu->smrs)
1119                 smmu->smrs[idx].valid = false;
1120 
1121         return true;
1122 }
1123 
1124 static int arm_smmu_master_alloc_smes(struct device *dev)
1125 {
1126         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1127         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1128         struct arm_smmu_device *smmu = cfg->smmu;
1129         struct arm_smmu_smr *smrs = smmu->smrs;
1130         struct iommu_group *group;
1131         int i, idx, ret;
1132 
1133         mutex_lock(&smmu->stream_map_mutex);
1134         /* Figure out a viable stream map entry allocation */
1135         for_each_cfg_sme(fwspec, i, idx) {
1136                 u16 sid = fwspec->ids[i];
1137                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1138 
1139                 if (idx != INVALID_SMENDX) {
1140                         ret = -EEXIST;
1141                         goto out_err;
1142                 }
1143 
1144                 ret = arm_smmu_find_sme(smmu, sid, mask);
1145                 if (ret < 0)
1146                         goto out_err;
1147 
1148                 idx = ret;
1149                 if (smrs && smmu->s2crs[idx].count == 0) {
1150                         smrs[idx].id = sid;
1151                         smrs[idx].mask = mask;
1152                         smrs[idx].valid = true;
1153                 }
1154                 smmu->s2crs[idx].count++;
1155                 cfg->smendx[i] = (s16)idx;
1156         }
1157 
1158         group = iommu_group_get_for_dev(dev);
1159         if (!group)
1160                 group = ERR_PTR(-ENOMEM);
1161         if (IS_ERR(group)) {
1162                 ret = PTR_ERR(group);
1163                 goto out_err;
1164         }
1165         iommu_group_put(group);
1166 
1167         /* It worked! Now, poke the actual hardware */
1168         for_each_cfg_sme(fwspec, i, idx) {
1169                 arm_smmu_write_sme(smmu, idx);
1170                 smmu->s2crs[idx].group = group;
1171         }
1172 
1173         mutex_unlock(&smmu->stream_map_mutex);
1174         return 0;
1175 
1176 out_err:
1177         while (i--) {
1178                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1179                 cfg->smendx[i] = INVALID_SMENDX;
1180         }
1181         mutex_unlock(&smmu->stream_map_mutex);
1182         return ret;
1183 }
1184 
1185 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1186 {
1187         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1188         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1189         int i, idx;
1190 
1191         mutex_lock(&smmu->stream_map_mutex);
1192         for_each_cfg_sme(fwspec, i, idx) {
1193                 if (arm_smmu_free_sme(smmu, idx))
1194                         arm_smmu_write_sme(smmu, idx);
1195                 cfg->smendx[i] = INVALID_SMENDX;
1196         }
1197         mutex_unlock(&smmu->stream_map_mutex);
1198 }
1199 
1200 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1201                                       struct iommu_fwspec *fwspec)
1202 {
1203         struct arm_smmu_device *smmu = smmu_domain->smmu;
1204         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1205         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1206         u8 cbndx = smmu_domain->cfg.cbndx;
1207         int i, idx;
1208 
1209         for_each_cfg_sme(fwspec, i, idx) {
1210                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1211                         continue;
1212 
1213                 s2cr[idx].type = type;
1214                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1215                 s2cr[idx].cbndx = cbndx;
1216                 arm_smmu_write_s2cr(smmu, idx);
1217         }
1218         return 0;
1219 }
1220 
1221 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1222 {
1223         int ret;
1224         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1225         struct arm_smmu_device *smmu;
1226         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1227 
1228         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1229                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1230                 return -ENXIO;
1231         }
1232 
1233         /*
1234          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1235          * domains between of_xlate() and add_device() - we have no way to cope
1236          * with that, so until ARM gets converted to rely on groups and default
1237          * domains, just say no (but more politely than by dereferencing NULL).
1238          * This should be at least a WARN_ON once that's sorted.
1239          */
1240         if (!fwspec->iommu_priv)
1241                 return -ENODEV;
1242 
1243         smmu = fwspec_smmu(fwspec);
1244         /* Ensure that the domain is finalised */
1245         ret = arm_smmu_init_domain_context(domain, smmu);
1246         if (ret < 0)
1247                 return ret;
1248 
1249         /*
1250          * Sanity check the domain. We don't support domains across
1251          * different SMMUs.
1252          */
1253         if (smmu_domain->smmu != smmu) {
1254                 dev_err(dev,
1255                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1256                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1257                 return -EINVAL;
1258         }
1259 
1260         /* Looks ok, so add the device to the domain */
1261         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1262 }
1263 
1264 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1265                         phys_addr_t paddr, size_t size, int prot)
1266 {
1267         int ret;
1268         unsigned long flags;
1269         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1270         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1271 
1272         if (!ops)
1273                 return -ENODEV;
1274 
1275         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1276         ret = ops->map(ops, iova, paddr, size, prot);
1277         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1278         return ret;
1279 }
1280 
1281 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1282                              size_t size)
1283 {
1284         size_t ret;
1285         unsigned long flags;
1286         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1287         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1288 
1289         if (!ops)
1290                 return 0;
1291 
1292         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1293         ret = ops->unmap(ops, iova, size);
1294         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1295         return ret;
1296 }
1297 
1298 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1299                                               dma_addr_t iova)
1300 {
1301         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1302         struct arm_smmu_device *smmu = smmu_domain->smmu;
1303         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1304         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1305         struct device *dev = smmu->dev;
1306         void __iomem *cb_base;
1307         u32 tmp;
1308         u64 phys;
1309         unsigned long va;
1310 
1311         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1312 
1313         /* ATS1 registers can only be written atomically */
1314         va = iova & ~0xfffUL;
1315         if (smmu->version == ARM_SMMU_V2)
1316                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1317         else /* Register is only 32-bit in v1 */
1318                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1319 
1320         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1321                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1322                 dev_err(dev,
1323                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1324                         &iova);
1325                 return ops->iova_to_phys(ops, iova);
1326         }
1327 
1328         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1329         if (phys & CB_PAR_F) {
1330                 dev_err(dev, "translation fault!\n");
1331                 dev_err(dev, "PAR = 0x%llx\n", phys);
1332                 return 0;
1333         }
1334 
1335         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1336 }
1337 
1338 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1339                                         dma_addr_t iova)
1340 {
1341         phys_addr_t ret;
1342         unsigned long flags;
1343         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1345 
1346         if (!ops)
1347                 return 0;
1348 
1349         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1350         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1351                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1352                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1353         } else {
1354                 ret = ops->iova_to_phys(ops, iova);
1355         }
1356 
1357         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1358 
1359         return ret;
1360 }
1361 
1362 static bool arm_smmu_capable(enum iommu_cap cap)
1363 {
1364         switch (cap) {
1365         case IOMMU_CAP_CACHE_COHERENCY:
1366                 /*
1367                  * Return true here as the SMMU can always send out coherent
1368                  * requests.
1369                  */
1370                 return true;
1371         case IOMMU_CAP_INTR_REMAP:
1372                 return true; /* MSIs are just memory writes */
1373         case IOMMU_CAP_NOEXEC:
1374                 return true;
1375         default:
1376                 return false;
1377         }
1378 }
1379 
1380 static int arm_smmu_match_node(struct device *dev, void *data)
1381 {
1382         return dev->of_node == data;
1383 }
1384 
1385 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1386 {
1387         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1388                                                 np, arm_smmu_match_node);
1389         put_device(dev);
1390         return dev ? dev_get_drvdata(dev) : NULL;
1391 }
1392 
1393 static int arm_smmu_add_device(struct device *dev)
1394 {
1395         struct arm_smmu_device *smmu;
1396         struct arm_smmu_master_cfg *cfg;
1397         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1398         int i, ret;
1399 
1400         if (using_legacy_binding) {
1401                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1402                 fwspec = dev->iommu_fwspec;
1403                 if (ret)
1404                         goto out_free;
1405         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1406                 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1407         } else {
1408                 return -ENODEV;
1409         }
1410 
1411         ret = -EINVAL;
1412         for (i = 0; i < fwspec->num_ids; i++) {
1413                 u16 sid = fwspec->ids[i];
1414                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1415 
1416                 if (sid & ~smmu->streamid_mask) {
1417                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1418                                 sid, smmu->streamid_mask);
1419                         goto out_free;
1420                 }
1421                 if (mask & ~smmu->smr_mask_mask) {
1422                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1423                                 sid, smmu->smr_mask_mask);
1424                         goto out_free;
1425                 }
1426         }
1427 
1428         ret = -ENOMEM;
1429         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1430                       GFP_KERNEL);
1431         if (!cfg)
1432                 goto out_free;
1433 
1434         cfg->smmu = smmu;
1435         fwspec->iommu_priv = cfg;
1436         while (i--)
1437                 cfg->smendx[i] = INVALID_SMENDX;
1438 
1439         ret = arm_smmu_master_alloc_smes(dev);
1440         if (ret)
1441                 goto out_free;
1442 
1443         return 0;
1444 
1445 out_free:
1446         if (fwspec)
1447                 kfree(fwspec->iommu_priv);
1448         iommu_fwspec_free(dev);
1449         return ret;
1450 }
1451 
1452 static void arm_smmu_remove_device(struct device *dev)
1453 {
1454         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1455 
1456         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1457                 return;
1458 
1459         arm_smmu_master_free_smes(fwspec);
1460         iommu_group_remove_device(dev);
1461         kfree(fwspec->iommu_priv);
1462         iommu_fwspec_free(dev);
1463 }
1464 
1465 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1466 {
1467         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1468         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1469         struct iommu_group *group = NULL;
1470         int i, idx;
1471 
1472         for_each_cfg_sme(fwspec, i, idx) {
1473                 if (group && smmu->s2crs[idx].group &&
1474                     group != smmu->s2crs[idx].group)
1475                         return ERR_PTR(-EINVAL);
1476 
1477                 group = smmu->s2crs[idx].group;
1478         }
1479 
1480         if (group)
1481                 return group;
1482 
1483         if (dev_is_pci(dev))
1484                 group = pci_device_group(dev);
1485         else
1486                 group = generic_device_group(dev);
1487 
1488         return group;
1489 }
1490 
1491 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1492                                     enum iommu_attr attr, void *data)
1493 {
1494         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1495 
1496         switch (attr) {
1497         case DOMAIN_ATTR_NESTING:
1498                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1499                 return 0;
1500         default:
1501                 return -ENODEV;
1502         }
1503 }
1504 
1505 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1506                                     enum iommu_attr attr, void *data)
1507 {
1508         int ret = 0;
1509         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1510 
1511         mutex_lock(&smmu_domain->init_mutex);
1512 
1513         switch (attr) {
1514         case DOMAIN_ATTR_NESTING:
1515                 if (smmu_domain->smmu) {
1516                         ret = -EPERM;
1517                         goto out_unlock;
1518                 }
1519 
1520                 if (*(int *)data)
1521                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1522                 else
1523                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1524 
1525                 break;
1526         default:
1527                 ret = -ENODEV;
1528         }
1529 
1530 out_unlock:
1531         mutex_unlock(&smmu_domain->init_mutex);
1532         return ret;
1533 }
1534 
1535 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1536 {
1537         u32 fwid = 0;
1538 
1539         if (args->args_count > 0)
1540                 fwid |= (u16)args->args[0];
1541 
1542         if (args->args_count > 1)
1543                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1544 
1545         return iommu_fwspec_add_ids(dev, &fwid, 1);
1546 }
1547 
1548 static struct iommu_ops arm_smmu_ops = {
1549         .capable                = arm_smmu_capable,
1550         .domain_alloc           = arm_smmu_domain_alloc,
1551         .domain_free            = arm_smmu_domain_free,
1552         .attach_dev             = arm_smmu_attach_dev,
1553         .map                    = arm_smmu_map,
1554         .unmap                  = arm_smmu_unmap,
1555         .map_sg                 = default_iommu_map_sg,
1556         .iova_to_phys           = arm_smmu_iova_to_phys,
1557         .add_device             = arm_smmu_add_device,
1558         .remove_device          = arm_smmu_remove_device,
1559         .device_group           = arm_smmu_device_group,
1560         .domain_get_attr        = arm_smmu_domain_get_attr,
1561         .domain_set_attr        = arm_smmu_domain_set_attr,
1562         .of_xlate               = arm_smmu_of_xlate,
1563         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1564 };
1565 
1566 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1567 {
1568         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1569         void __iomem *cb_base;
1570         int i;
1571         u32 reg, major;
1572 
1573         /* clear global FSR */
1574         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1575         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1576 
1577         /*
1578          * Reset stream mapping groups: Initial values mark all SMRn as
1579          * invalid and all S2CRn as bypass unless overridden.
1580          */
1581         for (i = 0; i < smmu->num_mapping_groups; ++i)
1582                 arm_smmu_write_sme(smmu, i);
1583 
1584         /*
1585          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1586          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1587          * bit is only present in MMU-500r2 onwards.
1588          */
1589         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1590         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1591         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1592                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1593                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1594                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1595         }
1596 
1597         /* Make sure all context banks are disabled and clear CB_FSR  */
1598         for (i = 0; i < smmu->num_context_banks; ++i) {
1599                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1600                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1601                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1602                 /*
1603                  * Disable MMU-500's not-particularly-beneficial next-page
1604                  * prefetcher for the sake of errata #841119 and #826419.
1605                  */
1606                 if (smmu->model == ARM_MMU500) {
1607                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1608                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1609                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1610                 }
1611         }
1612 
1613         /* Invalidate the TLB, just in case */
1614         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1615         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1616 
1617         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1618 
1619         /* Enable fault reporting */
1620         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1621 
1622         /* Disable TLB broadcasting. */
1623         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1624 
1625         /* Enable client access, handling unmatched streams as appropriate */
1626         reg &= ~sCR0_CLIENTPD;
1627         if (disable_bypass)
1628                 reg |= sCR0_USFCFG;
1629         else
1630                 reg &= ~sCR0_USFCFG;
1631 
1632         /* Disable forced broadcasting */
1633         reg &= ~sCR0_FB;
1634 
1635         /* Don't upgrade barriers */
1636         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1637 
1638         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1639                 reg |= sCR0_VMID16EN;
1640 
1641         /* Push the button */
1642         __arm_smmu_tlb_sync(smmu);
1643         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1644 }
1645 
1646 static int arm_smmu_id_size_to_bits(int size)
1647 {
1648         switch (size) {
1649         case 0:
1650                 return 32;
1651         case 1:
1652                 return 36;
1653         case 2:
1654                 return 40;
1655         case 3:
1656                 return 42;
1657         case 4:
1658                 return 44;
1659         case 5:
1660         default:
1661                 return 48;
1662         }
1663 }
1664 
1665 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1666 {
1667         unsigned long size;
1668         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1669         u32 id;
1670         bool cttw_dt, cttw_reg;
1671         int i;
1672 
1673         dev_notice(smmu->dev, "probing hardware configuration...\n");
1674         dev_notice(smmu->dev, "SMMUv%d with:\n",
1675                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1676 
1677         /* ID0 */
1678         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1679 
1680         /* Restrict available stages based on module parameter */
1681         if (force_stage == 1)
1682                 id &= ~(ID0_S2TS | ID0_NTS);
1683         else if (force_stage == 2)
1684                 id &= ~(ID0_S1TS | ID0_NTS);
1685 
1686         if (id & ID0_S1TS) {
1687                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1688                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1689         }
1690 
1691         if (id & ID0_S2TS) {
1692                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1693                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1694         }
1695 
1696         if (id & ID0_NTS) {
1697                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1698                 dev_notice(smmu->dev, "\tnested translation\n");
1699         }
1700 
1701         if (!(smmu->features &
1702                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1703                 dev_err(smmu->dev, "\tno translation support!\n");
1704                 return -ENODEV;
1705         }
1706 
1707         if ((id & ID0_S1TS) &&
1708                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1709                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1710                 dev_notice(smmu->dev, "\taddress translation ops\n");
1711         }
1712 
1713         /*
1714          * In order for DMA API calls to work properly, we must defer to what
1715          * the DT says about coherency, regardless of what the hardware claims.
1716          * Fortunately, this also opens up a workaround for systems where the
1717          * ID register value has ended up configured incorrectly.
1718          */
1719         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1720         cttw_reg = !!(id & ID0_CTTW);
1721         if (cttw_dt)
1722                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1723         if (cttw_dt || cttw_reg)
1724                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1725                            cttw_dt ? "" : "non-");
1726         if (cttw_dt != cttw_reg)
1727                 dev_notice(smmu->dev,
1728                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1729 
1730         /* Max. number of entries we have for stream matching/indexing */
1731         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1732         smmu->streamid_mask = size - 1;
1733         if (id & ID0_SMS) {
1734                 u32 smr;
1735 
1736                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1737                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1738                 if (size == 0) {
1739                         dev_err(smmu->dev,
1740                                 "stream-matching supported, but no SMRs present!\n");
1741                         return -ENODEV;
1742                 }
1743 
1744                 /*
1745                  * SMR.ID bits may not be preserved if the corresponding MASK
1746                  * bits are set, so check each one separately. We can reject
1747                  * masters later if they try to claim IDs outside these masks.
1748                  */
1749                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1750                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1751                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1752                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1753 
1754                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1755                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1756                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1757                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1758 
1759                 /* Zero-initialised to mark as invalid */
1760                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1761                                           GFP_KERNEL);
1762                 if (!smmu->smrs)
1763                         return -ENOMEM;
1764 
1765                 dev_notice(smmu->dev,
1766                            "\tstream matching with %lu register groups, mask 0x%x",
1767                            size, smmu->smr_mask_mask);
1768         }
1769         /* s2cr->type == 0 means translation, so initialise explicitly */
1770         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1771                                          GFP_KERNEL);
1772         if (!smmu->s2crs)
1773                 return -ENOMEM;
1774         for (i = 0; i < size; i++)
1775                 smmu->s2crs[i] = s2cr_init_val;
1776 
1777         smmu->num_mapping_groups = size;
1778         mutex_init(&smmu->stream_map_mutex);
1779 
1780         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1781                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1782                 if (!(id & ID0_PTFS_NO_AARCH32S))
1783                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1784         }
1785 
1786         /* ID1 */
1787         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1788         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1789 
1790         /* Check for size mismatch of SMMU address space from mapped region */
1791         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1792         size *= 2 << smmu->pgshift;
1793         if (smmu->size != size)
1794                 dev_warn(smmu->dev,
1795                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1796                         size, smmu->size);
1797 
1798         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1799         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1800         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1801                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1802                 return -ENODEV;
1803         }
1804         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1805                    smmu->num_context_banks, smmu->num_s2_context_banks);
1806         /*
1807          * Cavium CN88xx erratum #27704.
1808          * Ensure ASID and VMID allocation is unique across all SMMUs in
1809          * the system.
1810          */
1811         if (smmu->model == CAVIUM_SMMUV2) {
1812                 smmu->cavium_id_base =
1813                         atomic_add_return(smmu->num_context_banks,
1814                                           &cavium_smmu_context_count);
1815                 smmu->cavium_id_base -= smmu->num_context_banks;
1816         }
1817 
1818         /* ID2 */
1819         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1820         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1821         smmu->ipa_size = size;
1822 
1823         /* The output mask is also applied for bypass */
1824         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1825         smmu->pa_size = size;
1826 
1827         if (id & ID2_VMID16)
1828                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1829 
1830         /*
1831          * What the page table walker can address actually depends on which
1832          * descriptor format is in use, but since a) we don't know that yet,
1833          * and b) it can vary per context bank, this will have to do...
1834          */
1835         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1836                 dev_warn(smmu->dev,
1837                          "failed to set DMA mask for table walker\n");
1838 
1839         if (smmu->version < ARM_SMMU_V2) {
1840                 smmu->va_size = smmu->ipa_size;
1841                 if (smmu->version == ARM_SMMU_V1_64K)
1842                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1843         } else {
1844                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1845                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1846                 if (id & ID2_PTFS_4K)
1847                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1848                 if (id & ID2_PTFS_16K)
1849                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1850                 if (id & ID2_PTFS_64K)
1851                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1852         }
1853 
1854         /* Now we've corralled the various formats, what'll it do? */
1855         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1856                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1857         if (smmu->features &
1858             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1859                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1860         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1861                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1862         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1863                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1864 
1865         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1866                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1867         else
1868                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1869         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1870                    smmu->pgsize_bitmap);
1871 
1872 
1873         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1874                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1875                            smmu->va_size, smmu->ipa_size);
1876 
1877         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1878                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1879                            smmu->ipa_size, smmu->pa_size);
1880 
1881         return 0;
1882 }
1883 
1884 struct arm_smmu_match_data {
1885         enum arm_smmu_arch_version version;
1886         enum arm_smmu_implementation model;
1887 };
1888 
1889 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1890 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1891 
1892 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1893 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1894 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1895 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1896 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1897 
1898 static const struct of_device_id arm_smmu_of_match[] = {
1899         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1900         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1901         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1902         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1903         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1904         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1905         { },
1906 };
1907 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1908 
1909 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1910 {
1911         const struct arm_smmu_match_data *data;
1912         struct resource *res;
1913         struct arm_smmu_device *smmu;
1914         struct device *dev = &pdev->dev;
1915         int num_irqs, i, err;
1916         bool legacy_binding;
1917 
1918         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1919         if (legacy_binding && !using_generic_binding) {
1920                 if (!using_legacy_binding)
1921                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1922                 using_legacy_binding = true;
1923         } else if (!legacy_binding && !using_legacy_binding) {
1924                 using_generic_binding = true;
1925         } else {
1926                 dev_err(dev, "not probing due to mismatched DT properties\n");
1927                 return -ENODEV;
1928         }
1929 
1930         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1931         if (!smmu) {
1932                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1933                 return -ENOMEM;
1934         }
1935         smmu->dev = dev;
1936 
1937         data = of_device_get_match_data(dev);
1938         smmu->version = data->version;
1939         smmu->model = data->model;
1940 
1941         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1942         smmu->base = devm_ioremap_resource(dev, res);
1943         if (IS_ERR(smmu->base))
1944                 return PTR_ERR(smmu->base);
1945         smmu->size = resource_size(res);
1946 
1947         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1948                                  &smmu->num_global_irqs)) {
1949                 dev_err(dev, "missing #global-interrupts property\n");
1950                 return -ENODEV;
1951         }
1952 
1953         num_irqs = 0;
1954         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1955                 num_irqs++;
1956                 if (num_irqs > smmu->num_global_irqs)
1957                         smmu->num_context_irqs++;
1958         }
1959 
1960         if (!smmu->num_context_irqs) {
1961                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1962                         num_irqs, smmu->num_global_irqs + 1);
1963                 return -ENODEV;
1964         }
1965 
1966         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1967                                   GFP_KERNEL);
1968         if (!smmu->irqs) {
1969                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1970                 return -ENOMEM;
1971         }
1972 
1973         for (i = 0; i < num_irqs; ++i) {
1974                 int irq = platform_get_irq(pdev, i);
1975 
1976                 if (irq < 0) {
1977                         dev_err(dev, "failed to get irq index %d\n", i);
1978                         return -ENODEV;
1979                 }
1980                 smmu->irqs[i] = irq;
1981         }
1982 
1983         err = arm_smmu_device_cfg_probe(smmu);
1984         if (err)
1985                 return err;
1986 
1987         parse_driver_options(smmu);
1988 
1989         if (smmu->version == ARM_SMMU_V2 &&
1990             smmu->num_context_banks != smmu->num_context_irqs) {
1991                 dev_err(dev,
1992                         "found only %d context interrupt(s) but %d required\n",
1993                         smmu->num_context_irqs, smmu->num_context_banks);
1994                 return -ENODEV;
1995         }
1996 
1997         for (i = 0; i < smmu->num_global_irqs; ++i) {
1998                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1999                                        arm_smmu_global_fault,
2000                                        IRQF_SHARED,
2001                                        "arm-smmu global fault",
2002                                        smmu);
2003                 if (err) {
2004                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2005                                 i, smmu->irqs[i]);
2006                         return err;
2007                 }
2008         }
2009 
2010         of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
2011         platform_set_drvdata(pdev, smmu);
2012         arm_smmu_device_reset(smmu);
2013 
2014         /* Oh, for a proper bus abstraction */
2015         if (!iommu_present(&platform_bus_type))
2016                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2017 #ifdef CONFIG_ARM_AMBA
2018         if (!iommu_present(&amba_bustype))
2019                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2020 #endif
2021 #ifdef CONFIG_PCI
2022         if (!iommu_present(&pci_bus_type)) {
2023                 pci_request_acs();
2024                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2025         }
2026 #endif
2027         return 0;
2028 }
2029 
2030 static int arm_smmu_device_remove(struct platform_device *pdev)
2031 {
2032         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2033 
2034         if (!smmu)
2035                 return -ENODEV;
2036 
2037         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2038                 dev_err(&pdev->dev, "removing device with active domains!\n");
2039 
2040         /* Turn the thing off */
2041         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2042         return 0;
2043 }
2044 
2045 static struct platform_driver arm_smmu_driver = {
2046         .driver = {
2047                 .name           = "arm-smmu",
2048                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2049         },
2050         .probe  = arm_smmu_device_dt_probe,
2051         .remove = arm_smmu_device_remove,
2052 };
2053 
2054 static int __init arm_smmu_init(void)
2055 {
2056         static bool registered;
2057         int ret = 0;
2058 
2059         if (!registered) {
2060                 ret = platform_driver_register(&arm_smmu_driver);
2061                 registered = !ret;
2062         }
2063         return ret;
2064 }
2065 
2066 static void __exit arm_smmu_exit(void)
2067 {
2068         return platform_driver_unregister(&arm_smmu_driver);
2069 }
2070 
2071 subsys_initcall(arm_smmu_init);
2072 module_exit(arm_smmu_exit);
2073 
2074 static int __init arm_smmu_of_init(struct device_node *np)
2075 {
2076         int ret = arm_smmu_init();
2077 
2078         if (ret)
2079                 return ret;
2080 
2081         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2082                 return -ENODEV;
2083 
2084         return 0;
2085 }
2086 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2087 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2088 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2089 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2090 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2091 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2092 
2093 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2094 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2095 MODULE_LICENSE("GPL v2");
2096 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us