Version:  2.0.40 2.2.26 2.4.37 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7

Linux/drivers/iommu/arm-smmu.c

  1 /*
  2  * IOMMU API for ARM architected SMMU implementations.
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  *
  8  * This program is distributed in the hope that it will be useful,
  9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 11  * GNU General Public License for more details.
 12  *
 13  * You should have received a copy of the GNU General Public License
 14  * along with this program; if not, write to the Free Software
 15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 16  *
 17  * Copyright (C) 2013 ARM Limited
 18  *
 19  * Author: Will Deacon <will.deacon@arm.com>
 20  *
 21  * This driver currently supports:
 22  *      - SMMUv1 and v2 implementations
 23  *      - Stream-matching and stream-indexing
 24  *      - v7/v8 long-descriptor format
 25  *      - Non-secure access to the SMMU
 26  *      - Context fault reporting
 27  */
 28 
 29 #define pr_fmt(fmt) "arm-smmu: " fmt
 30 
 31 #include <linux/delay.h>
 32 #include <linux/dma-iommu.h>
 33 #include <linux/dma-mapping.h>
 34 #include <linux/err.h>
 35 #include <linux/interrupt.h>
 36 #include <linux/io.h>
 37 #include <linux/io-64-nonatomic-hi-lo.h>
 38 #include <linux/iommu.h>
 39 #include <linux/iopoll.h>
 40 #include <linux/module.h>
 41 #include <linux/of.h>
 42 #include <linux/of_address.h>
 43 #include <linux/pci.h>
 44 #include <linux/platform_device.h>
 45 #include <linux/slab.h>
 46 #include <linux/spinlock.h>
 47 
 48 #include <linux/amba/bus.h>
 49 
 50 #include "io-pgtable.h"
 51 
 52 /* Maximum number of stream IDs assigned to a single device */
 53 #define MAX_MASTER_STREAMIDS            128
 54 
 55 /* Maximum number of context banks per SMMU */
 56 #define ARM_SMMU_MAX_CBS                128
 57 
 58 /* Maximum number of mapping groups per SMMU */
 59 #define ARM_SMMU_MAX_SMRS               128
 60 
 61 /* SMMU global address space */
 62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
 63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
 64 
 65 /*
 66  * SMMU global address space with conditional offset to access secure
 67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
 68  * nsGFSYNR0: 0x450)
 69  */
 70 #define ARM_SMMU_GR0_NS(smmu)                                           \
 71         ((smmu)->base +                                                 \
 72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
 73                         ? 0x400 : 0))
 74 
 75 /*
 76  * Some 64-bit registers only make sense to write atomically, but in such
 77  * cases all the data relevant to AArch32 formats lies within the lower word,
 78  * therefore this actually makes more sense than it might first appear.
 79  */
 80 #ifdef CONFIG_64BIT
 81 #define smmu_write_atomic_lq            writeq_relaxed
 82 #else
 83 #define smmu_write_atomic_lq            writel_relaxed
 84 #endif
 85 
 86 /* Configuration registers */
 87 #define ARM_SMMU_GR0_sCR0               0x0
 88 #define sCR0_CLIENTPD                   (1 << 0)
 89 #define sCR0_GFRE                       (1 << 1)
 90 #define sCR0_GFIE                       (1 << 2)
 91 #define sCR0_GCFGFRE                    (1 << 4)
 92 #define sCR0_GCFGFIE                    (1 << 5)
 93 #define sCR0_USFCFG                     (1 << 10)
 94 #define sCR0_VMIDPNE                    (1 << 11)
 95 #define sCR0_PTM                        (1 << 12)
 96 #define sCR0_FB                         (1 << 13)
 97 #define sCR0_VMID16EN                   (1 << 31)
 98 #define sCR0_BSU_SHIFT                  14
 99 #define sCR0_BSU_MASK                   0x3
100 
101 /* Auxiliary Configuration register */
102 #define ARM_SMMU_GR0_sACR               0x10
103 
104 /* Identification registers */
105 #define ARM_SMMU_GR0_ID0                0x20
106 #define ARM_SMMU_GR0_ID1                0x24
107 #define ARM_SMMU_GR0_ID2                0x28
108 #define ARM_SMMU_GR0_ID3                0x2c
109 #define ARM_SMMU_GR0_ID4                0x30
110 #define ARM_SMMU_GR0_ID5                0x34
111 #define ARM_SMMU_GR0_ID6                0x38
112 #define ARM_SMMU_GR0_ID7                0x3c
113 #define ARM_SMMU_GR0_sGFSR              0x48
114 #define ARM_SMMU_GR0_sGFSYNR0           0x50
115 #define ARM_SMMU_GR0_sGFSYNR1           0x54
116 #define ARM_SMMU_GR0_sGFSYNR2           0x58
117 
118 #define ID0_S1TS                        (1 << 30)
119 #define ID0_S2TS                        (1 << 29)
120 #define ID0_NTS                         (1 << 28)
121 #define ID0_SMS                         (1 << 27)
122 #define ID0_ATOSNS                      (1 << 26)
123 #define ID0_PTFS_NO_AARCH32             (1 << 25)
124 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
125 #define ID0_CTTW                        (1 << 14)
126 #define ID0_NUMIRPT_SHIFT               16
127 #define ID0_NUMIRPT_MASK                0xff
128 #define ID0_NUMSIDB_SHIFT               9
129 #define ID0_NUMSIDB_MASK                0xf
130 #define ID0_NUMSMRG_SHIFT               0
131 #define ID0_NUMSMRG_MASK                0xff
132 
133 #define ID1_PAGESIZE                    (1 << 31)
134 #define ID1_NUMPAGENDXB_SHIFT           28
135 #define ID1_NUMPAGENDXB_MASK            7
136 #define ID1_NUMS2CB_SHIFT               16
137 #define ID1_NUMS2CB_MASK                0xff
138 #define ID1_NUMCB_SHIFT                 0
139 #define ID1_NUMCB_MASK                  0xff
140 
141 #define ID2_OAS_SHIFT                   4
142 #define ID2_OAS_MASK                    0xf
143 #define ID2_IAS_SHIFT                   0
144 #define ID2_IAS_MASK                    0xf
145 #define ID2_UBS_SHIFT                   8
146 #define ID2_UBS_MASK                    0xf
147 #define ID2_PTFS_4K                     (1 << 12)
148 #define ID2_PTFS_16K                    (1 << 13)
149 #define ID2_PTFS_64K                    (1 << 14)
150 #define ID2_VMID16                      (1 << 15)
151 
152 #define ID7_MAJOR_SHIFT                 4
153 #define ID7_MAJOR_MASK                  0xf
154 
155 /* Global TLB invalidation */
156 #define ARM_SMMU_GR0_TLBIVMID           0x64
157 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
158 #define ARM_SMMU_GR0_TLBIALLH           0x6c
159 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
160 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
161 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
162 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
163 
164 /* Stream mapping registers */
165 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
166 #define SMR_VALID                       (1 << 31)
167 #define SMR_MASK_SHIFT                  16
168 #define SMR_MASK_MASK                   0x7fff
169 #define SMR_ID_SHIFT                    0
170 #define SMR_ID_MASK                     0x7fff
171 
172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
173 #define S2CR_CBNDX_SHIFT                0
174 #define S2CR_CBNDX_MASK                 0xff
175 #define S2CR_TYPE_SHIFT                 16
176 #define S2CR_TYPE_MASK                  0x3
177 #define S2CR_TYPE_TRANS                 (0 << S2CR_TYPE_SHIFT)
178 #define S2CR_TYPE_BYPASS                (1 << S2CR_TYPE_SHIFT)
179 #define S2CR_TYPE_FAULT                 (2 << S2CR_TYPE_SHIFT)
180 
181 #define S2CR_PRIVCFG_SHIFT              24
182 #define S2CR_PRIVCFG_UNPRIV             (2 << S2CR_PRIVCFG_SHIFT)
183 
184 /* Context bank attribute registers */
185 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
186 #define CBAR_VMID_SHIFT                 0
187 #define CBAR_VMID_MASK                  0xff
188 #define CBAR_S1_BPSHCFG_SHIFT           8
189 #define CBAR_S1_BPSHCFG_MASK            3
190 #define CBAR_S1_BPSHCFG_NSH             3
191 #define CBAR_S1_MEMATTR_SHIFT           12
192 #define CBAR_S1_MEMATTR_MASK            0xf
193 #define CBAR_S1_MEMATTR_WB              0xf
194 #define CBAR_TYPE_SHIFT                 16
195 #define CBAR_TYPE_MASK                  0x3
196 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
197 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
198 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
199 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
200 #define CBAR_IRPTNDX_SHIFT              24
201 #define CBAR_IRPTNDX_MASK               0xff
202 
203 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
204 #define CBA2R_RW64_32BIT                (0 << 0)
205 #define CBA2R_RW64_64BIT                (1 << 0)
206 #define CBA2R_VMID_SHIFT                16
207 #define CBA2R_VMID_MASK                 0xffff
208 
209 /* Translation context bank */
210 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
211 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
212 
213 #define ARM_SMMU_CB_SCTLR               0x0
214 #define ARM_SMMU_CB_ACTLR               0x4
215 #define ARM_SMMU_CB_RESUME              0x8
216 #define ARM_SMMU_CB_TTBCR2              0x10
217 #define ARM_SMMU_CB_TTBR0               0x20
218 #define ARM_SMMU_CB_TTBR1               0x28
219 #define ARM_SMMU_CB_TTBCR               0x30
220 #define ARM_SMMU_CB_S1_MAIR0            0x38
221 #define ARM_SMMU_CB_S1_MAIR1            0x3c
222 #define ARM_SMMU_CB_PAR                 0x50
223 #define ARM_SMMU_CB_FSR                 0x58
224 #define ARM_SMMU_CB_FAR                 0x60
225 #define ARM_SMMU_CB_FSYNR0              0x68
226 #define ARM_SMMU_CB_S1_TLBIVA           0x600
227 #define ARM_SMMU_CB_S1_TLBIASID         0x610
228 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
229 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
230 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
231 #define ARM_SMMU_CB_ATS1PR              0x800
232 #define ARM_SMMU_CB_ATSR                0x8f0
233 
234 #define SCTLR_S1_ASIDPNE                (1 << 12)
235 #define SCTLR_CFCFG                     (1 << 7)
236 #define SCTLR_CFIE                      (1 << 6)
237 #define SCTLR_CFRE                      (1 << 5)
238 #define SCTLR_E                         (1 << 4)
239 #define SCTLR_AFE                       (1 << 2)
240 #define SCTLR_TRE                       (1 << 1)
241 #define SCTLR_M                         (1 << 0)
242 #define SCTLR_EAE_SBOP                  (SCTLR_AFE | SCTLR_TRE)
243 
244 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
245 
246 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
247 
248 #define CB_PAR_F                        (1 << 0)
249 
250 #define ATSR_ACTIVE                     (1 << 0)
251 
252 #define RESUME_RETRY                    (0 << 0)
253 #define RESUME_TERMINATE                (1 << 0)
254 
255 #define TTBCR2_SEP_SHIFT                15
256 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
257 
258 #define TTBRn_ASID_SHIFT                48
259 
260 #define FSR_MULTI                       (1 << 31)
261 #define FSR_SS                          (1 << 30)
262 #define FSR_UUT                         (1 << 8)
263 #define FSR_ASF                         (1 << 7)
264 #define FSR_TLBLKF                      (1 << 6)
265 #define FSR_TLBMCF                      (1 << 5)
266 #define FSR_EF                          (1 << 4)
267 #define FSR_PF                          (1 << 3)
268 #define FSR_AFF                         (1 << 2)
269 #define FSR_TF                          (1 << 1)
270 
271 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
272                                          FSR_TLBMCF | FSR_TLBLKF)
273 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
274                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
275 
276 #define FSYNR0_WNR                      (1 << 4)
277 
278 static int force_stage;
279 module_param(force_stage, int, S_IRUGO);
280 MODULE_PARM_DESC(force_stage,
281         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
282 static bool disable_bypass;
283 module_param(disable_bypass, bool, S_IRUGO);
284 MODULE_PARM_DESC(disable_bypass,
285         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
286 
287 enum arm_smmu_arch_version {
288         ARM_SMMU_V1,
289         ARM_SMMU_V1_64K,
290         ARM_SMMU_V2,
291 };
292 
293 enum arm_smmu_implementation {
294         GENERIC_SMMU,
295         ARM_MMU500,
296         CAVIUM_SMMUV2,
297 };
298 
299 struct arm_smmu_smr {
300         u8                              idx;
301         u16                             mask;
302         u16                             id;
303 };
304 
305 struct arm_smmu_master_cfg {
306         int                             num_streamids;
307         u16                             streamids[MAX_MASTER_STREAMIDS];
308         struct arm_smmu_smr             *smrs;
309 };
310 
311 struct arm_smmu_master {
312         struct device_node              *of_node;
313         struct rb_node                  node;
314         struct arm_smmu_master_cfg      cfg;
315 };
316 
317 struct arm_smmu_device {
318         struct device                   *dev;
319 
320         void __iomem                    *base;
321         unsigned long                   size;
322         unsigned long                   pgshift;
323 
324 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
325 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
326 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
327 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
328 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
329 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
330 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
331 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
332 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
333 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
334 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
335 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
336         u32                             features;
337 
338 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
339         u32                             options;
340         enum arm_smmu_arch_version      version;
341         enum arm_smmu_implementation    model;
342 
343         u32                             num_context_banks;
344         u32                             num_s2_context_banks;
345         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
346         atomic_t                        irptndx;
347 
348         u32                             num_mapping_groups;
349         DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
350 
351         unsigned long                   va_size;
352         unsigned long                   ipa_size;
353         unsigned long                   pa_size;
354         unsigned long                   pgsize_bitmap;
355 
356         u32                             num_global_irqs;
357         u32                             num_context_irqs;
358         unsigned int                    *irqs;
359 
360         struct list_head                list;
361         struct rb_root                  masters;
362 
363         u32                             cavium_id_base; /* Specific to Cavium */
364 };
365 
366 enum arm_smmu_context_fmt {
367         ARM_SMMU_CTX_FMT_NONE,
368         ARM_SMMU_CTX_FMT_AARCH64,
369         ARM_SMMU_CTX_FMT_AARCH32_L,
370         ARM_SMMU_CTX_FMT_AARCH32_S,
371 };
372 
373 struct arm_smmu_cfg {
374         u8                              cbndx;
375         u8                              irptndx;
376         u32                             cbar;
377         enum arm_smmu_context_fmt       fmt;
378 };
379 #define INVALID_IRPTNDX                 0xff
380 
381 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
382 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
383 
384 enum arm_smmu_domain_stage {
385         ARM_SMMU_DOMAIN_S1 = 0,
386         ARM_SMMU_DOMAIN_S2,
387         ARM_SMMU_DOMAIN_NESTED,
388 };
389 
390 struct arm_smmu_domain {
391         struct arm_smmu_device          *smmu;
392         struct io_pgtable_ops           *pgtbl_ops;
393         spinlock_t                      pgtbl_lock;
394         struct arm_smmu_cfg             cfg;
395         enum arm_smmu_domain_stage      stage;
396         struct mutex                    init_mutex; /* Protects smmu pointer */
397         struct iommu_domain             domain;
398 };
399 
400 struct arm_smmu_phandle_args {
401         struct device_node *np;
402         int args_count;
403         uint32_t args[MAX_MASTER_STREAMIDS];
404 };
405 
406 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
407 static LIST_HEAD(arm_smmu_devices);
408 
409 struct arm_smmu_option_prop {
410         u32 opt;
411         const char *prop;
412 };
413 
414 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
415 
416 static struct arm_smmu_option_prop arm_smmu_options[] = {
417         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
418         { 0, NULL},
419 };
420 
421 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
422 {
423         return container_of(dom, struct arm_smmu_domain, domain);
424 }
425 
426 static void parse_driver_options(struct arm_smmu_device *smmu)
427 {
428         int i = 0;
429 
430         do {
431                 if (of_property_read_bool(smmu->dev->of_node,
432                                                 arm_smmu_options[i].prop)) {
433                         smmu->options |= arm_smmu_options[i].opt;
434                         dev_notice(smmu->dev, "option %s\n",
435                                 arm_smmu_options[i].prop);
436                 }
437         } while (arm_smmu_options[++i].opt);
438 }
439 
440 static struct device_node *dev_get_dev_node(struct device *dev)
441 {
442         if (dev_is_pci(dev)) {
443                 struct pci_bus *bus = to_pci_dev(dev)->bus;
444 
445                 while (!pci_is_root_bus(bus))
446                         bus = bus->parent;
447                 return bus->bridge->parent->of_node;
448         }
449 
450         return dev->of_node;
451 }
452 
453 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
454                                                 struct device_node *dev_node)
455 {
456         struct rb_node *node = smmu->masters.rb_node;
457 
458         while (node) {
459                 struct arm_smmu_master *master;
460 
461                 master = container_of(node, struct arm_smmu_master, node);
462 
463                 if (dev_node < master->of_node)
464                         node = node->rb_left;
465                 else if (dev_node > master->of_node)
466                         node = node->rb_right;
467                 else
468                         return master;
469         }
470 
471         return NULL;
472 }
473 
474 static struct arm_smmu_master_cfg *
475 find_smmu_master_cfg(struct device *dev)
476 {
477         struct arm_smmu_master_cfg *cfg = NULL;
478         struct iommu_group *group = iommu_group_get(dev);
479 
480         if (group) {
481                 cfg = iommu_group_get_iommudata(group);
482                 iommu_group_put(group);
483         }
484 
485         return cfg;
486 }
487 
488 static int insert_smmu_master(struct arm_smmu_device *smmu,
489                               struct arm_smmu_master *master)
490 {
491         struct rb_node **new, *parent;
492 
493         new = &smmu->masters.rb_node;
494         parent = NULL;
495         while (*new) {
496                 struct arm_smmu_master *this
497                         = container_of(*new, struct arm_smmu_master, node);
498 
499                 parent = *new;
500                 if (master->of_node < this->of_node)
501                         new = &((*new)->rb_left);
502                 else if (master->of_node > this->of_node)
503                         new = &((*new)->rb_right);
504                 else
505                         return -EEXIST;
506         }
507 
508         rb_link_node(&master->node, parent, new);
509         rb_insert_color(&master->node, &smmu->masters);
510         return 0;
511 }
512 
513 static int register_smmu_master(struct arm_smmu_device *smmu,
514                                 struct device *dev,
515                                 struct arm_smmu_phandle_args *masterspec)
516 {
517         int i;
518         struct arm_smmu_master *master;
519 
520         master = find_smmu_master(smmu, masterspec->np);
521         if (master) {
522                 dev_err(dev,
523                         "rejecting multiple registrations for master device %s\n",
524                         masterspec->np->name);
525                 return -EBUSY;
526         }
527 
528         if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
529                 dev_err(dev,
530                         "reached maximum number (%d) of stream IDs for master device %s\n",
531                         MAX_MASTER_STREAMIDS, masterspec->np->name);
532                 return -ENOSPC;
533         }
534 
535         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
536         if (!master)
537                 return -ENOMEM;
538 
539         master->of_node                 = masterspec->np;
540         master->cfg.num_streamids       = masterspec->args_count;
541 
542         for (i = 0; i < master->cfg.num_streamids; ++i) {
543                 u16 streamid = masterspec->args[i];
544 
545                 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
546                      (streamid >= smmu->num_mapping_groups)) {
547                         dev_err(dev,
548                                 "stream ID for master device %s greater than maximum allowed (%d)\n",
549                                 masterspec->np->name, smmu->num_mapping_groups);
550                         return -ERANGE;
551                 }
552                 master->cfg.streamids[i] = streamid;
553         }
554         return insert_smmu_master(smmu, master);
555 }
556 
557 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
558 {
559         struct arm_smmu_device *smmu;
560         struct arm_smmu_master *master = NULL;
561         struct device_node *dev_node = dev_get_dev_node(dev);
562 
563         spin_lock(&arm_smmu_devices_lock);
564         list_for_each_entry(smmu, &arm_smmu_devices, list) {
565                 master = find_smmu_master(smmu, dev_node);
566                 if (master)
567                         break;
568         }
569         spin_unlock(&arm_smmu_devices_lock);
570 
571         return master ? smmu : NULL;
572 }
573 
574 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
575 {
576         int idx;
577 
578         do {
579                 idx = find_next_zero_bit(map, end, start);
580                 if (idx == end)
581                         return -ENOSPC;
582         } while (test_and_set_bit(idx, map));
583 
584         return idx;
585 }
586 
587 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
588 {
589         clear_bit(idx, map);
590 }
591 
592 /* Wait for any pending TLB invalidations to complete */
593 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
594 {
595         int count = 0;
596         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
597 
598         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
599         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
600                & sTLBGSTATUS_GSACTIVE) {
601                 cpu_relax();
602                 if (++count == TLB_LOOP_TIMEOUT) {
603                         dev_err_ratelimited(smmu->dev,
604                         "TLB sync timed out -- SMMU may be deadlocked\n");
605                         return;
606                 }
607                 udelay(1);
608         }
609 }
610 
611 static void arm_smmu_tlb_sync(void *cookie)
612 {
613         struct arm_smmu_domain *smmu_domain = cookie;
614         __arm_smmu_tlb_sync(smmu_domain->smmu);
615 }
616 
617 static void arm_smmu_tlb_inv_context(void *cookie)
618 {
619         struct arm_smmu_domain *smmu_domain = cookie;
620         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621         struct arm_smmu_device *smmu = smmu_domain->smmu;
622         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
623         void __iomem *base;
624 
625         if (stage1) {
626                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
627                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
628                                base + ARM_SMMU_CB_S1_TLBIASID);
629         } else {
630                 base = ARM_SMMU_GR0(smmu);
631                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
632                                base + ARM_SMMU_GR0_TLBIVMID);
633         }
634 
635         __arm_smmu_tlb_sync(smmu);
636 }
637 
638 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
639                                           size_t granule, bool leaf, void *cookie)
640 {
641         struct arm_smmu_domain *smmu_domain = cookie;
642         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
643         struct arm_smmu_device *smmu = smmu_domain->smmu;
644         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
645         void __iomem *reg;
646 
647         if (stage1) {
648                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
649                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
650 
651                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
652                         iova &= ~12UL;
653                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
654                         do {
655                                 writel_relaxed(iova, reg);
656                                 iova += granule;
657                         } while (size -= granule);
658                 } else {
659                         iova >>= 12;
660                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
661                         do {
662                                 writeq_relaxed(iova, reg);
663                                 iova += granule >> 12;
664                         } while (size -= granule);
665                 }
666         } else if (smmu->version == ARM_SMMU_V2) {
667                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
668                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
669                               ARM_SMMU_CB_S2_TLBIIPAS2;
670                 iova >>= 12;
671                 do {
672                         smmu_write_atomic_lq(iova, reg);
673                         iova += granule >> 12;
674                 } while (size -= granule);
675         } else {
676                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
677                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
678         }
679 }
680 
681 static struct iommu_gather_ops arm_smmu_gather_ops = {
682         .tlb_flush_all  = arm_smmu_tlb_inv_context,
683         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
684         .tlb_sync       = arm_smmu_tlb_sync,
685 };
686 
687 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
688 {
689         int flags, ret;
690         u32 fsr, fsynr, resume;
691         unsigned long iova;
692         struct iommu_domain *domain = dev;
693         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
694         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
695         struct arm_smmu_device *smmu = smmu_domain->smmu;
696         void __iomem *cb_base;
697 
698         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
699         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
700 
701         if (!(fsr & FSR_FAULT))
702                 return IRQ_NONE;
703 
704         if (fsr & FSR_IGN)
705                 dev_err_ratelimited(smmu->dev,
706                                     "Unexpected context fault (fsr 0x%x)\n",
707                                     fsr);
708 
709         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
710         flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
711 
712         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
713         if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
714                 ret = IRQ_HANDLED;
715                 resume = RESUME_RETRY;
716         } else {
717                 dev_err_ratelimited(smmu->dev,
718                     "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
719                     iova, fsynr, cfg->cbndx);
720                 ret = IRQ_NONE;
721                 resume = RESUME_TERMINATE;
722         }
723 
724         /* Clear the faulting FSR */
725         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
726 
727         /* Retry or terminate any stalled transactions */
728         if (fsr & FSR_SS)
729                 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
730 
731         return ret;
732 }
733 
734 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
735 {
736         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
737         struct arm_smmu_device *smmu = dev;
738         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
739 
740         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
741         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
742         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
743         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
744 
745         if (!gfsr)
746                 return IRQ_NONE;
747 
748         dev_err_ratelimited(smmu->dev,
749                 "Unexpected global fault, this could be serious\n");
750         dev_err_ratelimited(smmu->dev,
751                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
752                 gfsr, gfsynr0, gfsynr1, gfsynr2);
753 
754         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
755         return IRQ_HANDLED;
756 }
757 
758 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
759                                        struct io_pgtable_cfg *pgtbl_cfg)
760 {
761         u32 reg;
762         u64 reg64;
763         bool stage1;
764         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
765         struct arm_smmu_device *smmu = smmu_domain->smmu;
766         void __iomem *cb_base, *gr1_base;
767 
768         gr1_base = ARM_SMMU_GR1(smmu);
769         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
770         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
771 
772         if (smmu->version > ARM_SMMU_V1) {
773                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
774                         reg = CBA2R_RW64_64BIT;
775                 else
776                         reg = CBA2R_RW64_32BIT;
777                 /* 16-bit VMIDs live in CBA2R */
778                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
779                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
780 
781                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
782         }
783 
784         /* CBAR */
785         reg = cfg->cbar;
786         if (smmu->version < ARM_SMMU_V2)
787                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
788 
789         /*
790          * Use the weakest shareability/memory types, so they are
791          * overridden by the ttbcr/pte.
792          */
793         if (stage1) {
794                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
795                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
796         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
797                 /* 8-bit VMIDs live in CBAR */
798                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
799         }
800         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
801 
802         /* TTBRs */
803         if (stage1) {
804                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
805 
806                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
807                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
808 
809                 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
810                 reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
811                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
812         } else {
813                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
814                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
815         }
816 
817         /* TTBCR */
818         if (stage1) {
819                 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
820                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
821                 if (smmu->version > ARM_SMMU_V1) {
822                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
823                         reg |= TTBCR2_SEP_UPSTREAM;
824                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
825                 }
826         } else {
827                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
828                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
829         }
830 
831         /* MAIRs (stage-1 only) */
832         if (stage1) {
833                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
834                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
835                 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
836                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
837         }
838 
839         /* SCTLR */
840         reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
841         if (stage1)
842                 reg |= SCTLR_S1_ASIDPNE;
843 #ifdef __BIG_ENDIAN
844         reg |= SCTLR_E;
845 #endif
846         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
847 }
848 
849 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
850                                         struct arm_smmu_device *smmu)
851 {
852         int irq, start, ret = 0;
853         unsigned long ias, oas;
854         struct io_pgtable_ops *pgtbl_ops;
855         struct io_pgtable_cfg pgtbl_cfg;
856         enum io_pgtable_fmt fmt;
857         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
858         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
859 
860         mutex_lock(&smmu_domain->init_mutex);
861         if (smmu_domain->smmu)
862                 goto out_unlock;
863 
864         /* We're bypassing these SIDs, so don't allocate an actual context */
865         if (domain->type == IOMMU_DOMAIN_DMA) {
866                 smmu_domain->smmu = smmu;
867                 goto out_unlock;
868         }
869 
870         /*
871          * Mapping the requested stage onto what we support is surprisingly
872          * complicated, mainly because the spec allows S1+S2 SMMUs without
873          * support for nested translation. That means we end up with the
874          * following table:
875          *
876          * Requested        Supported        Actual
877          *     S1               N              S1
878          *     S1             S1+S2            S1
879          *     S1               S2             S2
880          *     S1               S1             S1
881          *     N                N              N
882          *     N              S1+S2            S2
883          *     N                S2             S2
884          *     N                S1             S1
885          *
886          * Note that you can't actually request stage-2 mappings.
887          */
888         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
889                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
890         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
891                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
892 
893         /*
894          * Choosing a suitable context format is even more fiddly. Until we
895          * grow some way for the caller to express a preference, and/or move
896          * the decision into the io-pgtable code where it arguably belongs,
897          * just aim for the closest thing to the rest of the system, and hope
898          * that the hardware isn't esoteric enough that we can't assume AArch64
899          * support to be a superset of AArch32 support...
900          */
901         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
902                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
903         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
904             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
905                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
906                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
907                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
908 
909         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
910                 ret = -EINVAL;
911                 goto out_unlock;
912         }
913 
914         switch (smmu_domain->stage) {
915         case ARM_SMMU_DOMAIN_S1:
916                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
917                 start = smmu->num_s2_context_banks;
918                 ias = smmu->va_size;
919                 oas = smmu->ipa_size;
920                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
921                         fmt = ARM_64_LPAE_S1;
922                 } else {
923                         fmt = ARM_32_LPAE_S1;
924                         ias = min(ias, 32UL);
925                         oas = min(oas, 40UL);
926                 }
927                 break;
928         case ARM_SMMU_DOMAIN_NESTED:
929                 /*
930                  * We will likely want to change this if/when KVM gets
931                  * involved.
932                  */
933         case ARM_SMMU_DOMAIN_S2:
934                 cfg->cbar = CBAR_TYPE_S2_TRANS;
935                 start = 0;
936                 ias = smmu->ipa_size;
937                 oas = smmu->pa_size;
938                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
939                         fmt = ARM_64_LPAE_S2;
940                 } else {
941                         fmt = ARM_32_LPAE_S2;
942                         ias = min(ias, 40UL);
943                         oas = min(oas, 40UL);
944                 }
945                 break;
946         default:
947                 ret = -EINVAL;
948                 goto out_unlock;
949         }
950 
951         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
952                                       smmu->num_context_banks);
953         if (ret < 0)
954                 goto out_unlock;
955 
956         cfg->cbndx = ret;
957         if (smmu->version < ARM_SMMU_V2) {
958                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
959                 cfg->irptndx %= smmu->num_context_irqs;
960         } else {
961                 cfg->irptndx = cfg->cbndx;
962         }
963 
964         pgtbl_cfg = (struct io_pgtable_cfg) {
965                 .pgsize_bitmap  = smmu->pgsize_bitmap,
966                 .ias            = ias,
967                 .oas            = oas,
968                 .tlb            = &arm_smmu_gather_ops,
969                 .iommu_dev      = smmu->dev,
970         };
971 
972         smmu_domain->smmu = smmu;
973         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
974         if (!pgtbl_ops) {
975                 ret = -ENOMEM;
976                 goto out_clear_smmu;
977         }
978 
979         /* Update the domain's page sizes to reflect the page table format */
980         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
981 
982         /* Initialise the context bank with our page table cfg */
983         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
984 
985         /*
986          * Request context fault interrupt. Do this last to avoid the
987          * handler seeing a half-initialised domain state.
988          */
989         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
990         ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
991                           "arm-smmu-context-fault", domain);
992         if (ret < 0) {
993                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
994                         cfg->irptndx, irq);
995                 cfg->irptndx = INVALID_IRPTNDX;
996         }
997 
998         mutex_unlock(&smmu_domain->init_mutex);
999 
1000         /* Publish page table ops for map/unmap */
1001         smmu_domain->pgtbl_ops = pgtbl_ops;
1002         return 0;
1003 
1004 out_clear_smmu:
1005         smmu_domain->smmu = NULL;
1006 out_unlock:
1007         mutex_unlock(&smmu_domain->init_mutex);
1008         return ret;
1009 }
1010 
1011 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1012 {
1013         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1014         struct arm_smmu_device *smmu = smmu_domain->smmu;
1015         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1016         void __iomem *cb_base;
1017         int irq;
1018 
1019         if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
1020                 return;
1021 
1022         /*
1023          * Disable the context bank and free the page tables before freeing
1024          * it.
1025          */
1026         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1027         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1028 
1029         if (cfg->irptndx != INVALID_IRPTNDX) {
1030                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1031                 free_irq(irq, domain);
1032         }
1033 
1034         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1035         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1036 }
1037 
1038 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1039 {
1040         struct arm_smmu_domain *smmu_domain;
1041 
1042         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1043                 return NULL;
1044         /*
1045          * Allocate the domain and initialise some of its data structures.
1046          * We can't really do anything meaningful until we've added a
1047          * master.
1048          */
1049         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1050         if (!smmu_domain)
1051                 return NULL;
1052 
1053         if (type == IOMMU_DOMAIN_DMA &&
1054             iommu_get_dma_cookie(&smmu_domain->domain)) {
1055                 kfree(smmu_domain);
1056                 return NULL;
1057         }
1058 
1059         mutex_init(&smmu_domain->init_mutex);
1060         spin_lock_init(&smmu_domain->pgtbl_lock);
1061 
1062         return &smmu_domain->domain;
1063 }
1064 
1065 static void arm_smmu_domain_free(struct iommu_domain *domain)
1066 {
1067         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1068 
1069         /*
1070          * Free the domain resources. We assume that all devices have
1071          * already been detached.
1072          */
1073         iommu_put_dma_cookie(domain);
1074         arm_smmu_destroy_domain_context(domain);
1075         kfree(smmu_domain);
1076 }
1077 
1078 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1079                                           struct arm_smmu_master_cfg *cfg)
1080 {
1081         int i;
1082         struct arm_smmu_smr *smrs;
1083         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1084 
1085         if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1086                 return 0;
1087 
1088         if (cfg->smrs)
1089                 return -EEXIST;
1090 
1091         smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1092         if (!smrs) {
1093                 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1094                         cfg->num_streamids);
1095                 return -ENOMEM;
1096         }
1097 
1098         /* Allocate the SMRs on the SMMU */
1099         for (i = 0; i < cfg->num_streamids; ++i) {
1100                 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1101                                                   smmu->num_mapping_groups);
1102                 if (idx < 0) {
1103                         dev_err(smmu->dev, "failed to allocate free SMR\n");
1104                         goto err_free_smrs;
1105                 }
1106 
1107                 smrs[i] = (struct arm_smmu_smr) {
1108                         .idx    = idx,
1109                         .mask   = 0, /* We don't currently share SMRs */
1110                         .id     = cfg->streamids[i],
1111                 };
1112         }
1113 
1114         /* It worked! Now, poke the actual hardware */
1115         for (i = 0; i < cfg->num_streamids; ++i) {
1116                 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1117                           smrs[i].mask << SMR_MASK_SHIFT;
1118                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1119         }
1120 
1121         cfg->smrs = smrs;
1122         return 0;
1123 
1124 err_free_smrs:
1125         while (--i >= 0)
1126                 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1127         kfree(smrs);
1128         return -ENOSPC;
1129 }
1130 
1131 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1132                                       struct arm_smmu_master_cfg *cfg)
1133 {
1134         int i;
1135         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1136         struct arm_smmu_smr *smrs = cfg->smrs;
1137 
1138         if (!smrs)
1139                 return;
1140 
1141         /* Invalidate the SMRs before freeing back to the allocator */
1142         for (i = 0; i < cfg->num_streamids; ++i) {
1143                 u8 idx = smrs[i].idx;
1144 
1145                 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1146                 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1147         }
1148 
1149         cfg->smrs = NULL;
1150         kfree(smrs);
1151 }
1152 
1153 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1154                                       struct arm_smmu_master_cfg *cfg)
1155 {
1156         int i, ret;
1157         struct arm_smmu_device *smmu = smmu_domain->smmu;
1158         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1159 
1160         /*
1161          * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1162          * for all devices behind the SMMU. Note that we need to take
1163          * care configuring SMRs for devices both a platform_device and
1164          * and a PCI device (i.e. a PCI host controller)
1165          */
1166         if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1167                 return 0;
1168 
1169         /* Devices in an IOMMU group may already be configured */
1170         ret = arm_smmu_master_configure_smrs(smmu, cfg);
1171         if (ret)
1172                 return ret == -EEXIST ? 0 : ret;
1173 
1174         for (i = 0; i < cfg->num_streamids; ++i) {
1175                 u32 idx, s2cr;
1176 
1177                 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1178                 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1179                        (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1180                 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1181         }
1182 
1183         return 0;
1184 }
1185 
1186 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1187                                           struct arm_smmu_master_cfg *cfg)
1188 {
1189         int i;
1190         struct arm_smmu_device *smmu = smmu_domain->smmu;
1191         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1192 
1193         /* An IOMMU group is torn down by the first device to be removed */
1194         if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1195                 return;
1196 
1197         /*
1198          * We *must* clear the S2CR first, because freeing the SMR means
1199          * that it can be re-allocated immediately.
1200          */
1201         for (i = 0; i < cfg->num_streamids; ++i) {
1202                 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1203                 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1204 
1205                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1206         }
1207 
1208         arm_smmu_master_free_smrs(smmu, cfg);
1209 }
1210 
1211 static void arm_smmu_detach_dev(struct device *dev,
1212                                 struct arm_smmu_master_cfg *cfg)
1213 {
1214         struct iommu_domain *domain = dev->archdata.iommu;
1215         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216 
1217         dev->archdata.iommu = NULL;
1218         arm_smmu_domain_remove_master(smmu_domain, cfg);
1219 }
1220 
1221 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1222 {
1223         int ret;
1224         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1225         struct arm_smmu_device *smmu;
1226         struct arm_smmu_master_cfg *cfg;
1227 
1228         smmu = find_smmu_for_device(dev);
1229         if (!smmu) {
1230                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1231                 return -ENXIO;
1232         }
1233 
1234         /* Ensure that the domain is finalised */
1235         ret = arm_smmu_init_domain_context(domain, smmu);
1236         if (ret < 0)
1237                 return ret;
1238 
1239         /*
1240          * Sanity check the domain. We don't support domains across
1241          * different SMMUs.
1242          */
1243         if (smmu_domain->smmu != smmu) {
1244                 dev_err(dev,
1245                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1246                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1247                 return -EINVAL;
1248         }
1249 
1250         /* Looks ok, so add the device to the domain */
1251         cfg = find_smmu_master_cfg(dev);
1252         if (!cfg)
1253                 return -ENODEV;
1254 
1255         /* Detach the dev from its current domain */
1256         if (dev->archdata.iommu)
1257                 arm_smmu_detach_dev(dev, cfg);
1258 
1259         ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1260         if (!ret)
1261                 dev->archdata.iommu = domain;
1262         return ret;
1263 }
1264 
1265 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1266                         phys_addr_t paddr, size_t size, int prot)
1267 {
1268         int ret;
1269         unsigned long flags;
1270         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1271         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1272 
1273         if (!ops)
1274                 return -ENODEV;
1275 
1276         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1277         ret = ops->map(ops, iova, paddr, size, prot);
1278         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1279         return ret;
1280 }
1281 
1282 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1283                              size_t size)
1284 {
1285         size_t ret;
1286         unsigned long flags;
1287         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1288         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1289 
1290         if (!ops)
1291                 return 0;
1292 
1293         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1294         ret = ops->unmap(ops, iova, size);
1295         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1296         return ret;
1297 }
1298 
1299 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1300                                               dma_addr_t iova)
1301 {
1302         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1303         struct arm_smmu_device *smmu = smmu_domain->smmu;
1304         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1305         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1306         struct device *dev = smmu->dev;
1307         void __iomem *cb_base;
1308         u32 tmp;
1309         u64 phys;
1310         unsigned long va;
1311 
1312         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1313 
1314         /* ATS1 registers can only be written atomically */
1315         va = iova & ~0xfffUL;
1316         if (smmu->version == ARM_SMMU_V2)
1317                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1318         else /* Register is only 32-bit in v1 */
1319                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1320 
1321         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1322                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1323                 dev_err(dev,
1324                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1325                         &iova);
1326                 return ops->iova_to_phys(ops, iova);
1327         }
1328 
1329         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1330         if (phys & CB_PAR_F) {
1331                 dev_err(dev, "translation fault!\n");
1332                 dev_err(dev, "PAR = 0x%llx\n", phys);
1333                 return 0;
1334         }
1335 
1336         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1337 }
1338 
1339 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1340                                         dma_addr_t iova)
1341 {
1342         phys_addr_t ret;
1343         unsigned long flags;
1344         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1345         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1346 
1347         if (!ops)
1348                 return 0;
1349 
1350         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1351         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1352                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1353                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1354         } else {
1355                 ret = ops->iova_to_phys(ops, iova);
1356         }
1357 
1358         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1359 
1360         return ret;
1361 }
1362 
1363 static bool arm_smmu_capable(enum iommu_cap cap)
1364 {
1365         switch (cap) {
1366         case IOMMU_CAP_CACHE_COHERENCY:
1367                 /*
1368                  * Return true here as the SMMU can always send out coherent
1369                  * requests.
1370                  */
1371                 return true;
1372         case IOMMU_CAP_INTR_REMAP:
1373                 return true; /* MSIs are just memory writes */
1374         case IOMMU_CAP_NOEXEC:
1375                 return true;
1376         default:
1377                 return false;
1378         }
1379 }
1380 
1381 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1382 {
1383         *((u16 *)data) = alias;
1384         return 0; /* Continue walking */
1385 }
1386 
1387 static void __arm_smmu_release_pci_iommudata(void *data)
1388 {
1389         kfree(data);
1390 }
1391 
1392 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1393                                     struct iommu_group *group)
1394 {
1395         struct arm_smmu_master_cfg *cfg;
1396         u16 sid;
1397         int i;
1398 
1399         cfg = iommu_group_get_iommudata(group);
1400         if (!cfg) {
1401                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1402                 if (!cfg)
1403                         return -ENOMEM;
1404 
1405                 iommu_group_set_iommudata(group, cfg,
1406                                           __arm_smmu_release_pci_iommudata);
1407         }
1408 
1409         if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1410                 return -ENOSPC;
1411 
1412         /*
1413          * Assume Stream ID == Requester ID for now.
1414          * We need a way to describe the ID mappings in FDT.
1415          */
1416         pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1417         for (i = 0; i < cfg->num_streamids; ++i)
1418                 if (cfg->streamids[i] == sid)
1419                         break;
1420 
1421         /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1422         if (i == cfg->num_streamids)
1423                 cfg->streamids[cfg->num_streamids++] = sid;
1424 
1425         return 0;
1426 }
1427 
1428 static int arm_smmu_init_platform_device(struct device *dev,
1429                                          struct iommu_group *group)
1430 {
1431         struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1432         struct arm_smmu_master *master;
1433 
1434         if (!smmu)
1435                 return -ENODEV;
1436 
1437         master = find_smmu_master(smmu, dev->of_node);
1438         if (!master)
1439                 return -ENODEV;
1440 
1441         iommu_group_set_iommudata(group, &master->cfg, NULL);
1442 
1443         return 0;
1444 }
1445 
1446 static int arm_smmu_add_device(struct device *dev)
1447 {
1448         struct iommu_group *group;
1449 
1450         group = iommu_group_get_for_dev(dev);
1451         if (IS_ERR(group))
1452                 return PTR_ERR(group);
1453 
1454         iommu_group_put(group);
1455         return 0;
1456 }
1457 
1458 static void arm_smmu_remove_device(struct device *dev)
1459 {
1460         iommu_group_remove_device(dev);
1461 }
1462 
1463 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1464 {
1465         struct iommu_group *group;
1466         int ret;
1467 
1468         if (dev_is_pci(dev))
1469                 group = pci_device_group(dev);
1470         else
1471                 group = generic_device_group(dev);
1472 
1473         if (IS_ERR(group))
1474                 return group;
1475 
1476         if (dev_is_pci(dev))
1477                 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1478         else
1479                 ret = arm_smmu_init_platform_device(dev, group);
1480 
1481         if (ret) {
1482                 iommu_group_put(group);
1483                 group = ERR_PTR(ret);
1484         }
1485 
1486         return group;
1487 }
1488 
1489 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1490                                     enum iommu_attr attr, void *data)
1491 {
1492         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1493 
1494         switch (attr) {
1495         case DOMAIN_ATTR_NESTING:
1496                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1497                 return 0;
1498         default:
1499                 return -ENODEV;
1500         }
1501 }
1502 
1503 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1504                                     enum iommu_attr attr, void *data)
1505 {
1506         int ret = 0;
1507         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1508 
1509         mutex_lock(&smmu_domain->init_mutex);
1510 
1511         switch (attr) {
1512         case DOMAIN_ATTR_NESTING:
1513                 if (smmu_domain->smmu) {
1514                         ret = -EPERM;
1515                         goto out_unlock;
1516                 }
1517 
1518                 if (*(int *)data)
1519                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1520                 else
1521                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1522 
1523                 break;
1524         default:
1525                 ret = -ENODEV;
1526         }
1527 
1528 out_unlock:
1529         mutex_unlock(&smmu_domain->init_mutex);
1530         return ret;
1531 }
1532 
1533 static struct iommu_ops arm_smmu_ops = {
1534         .capable                = arm_smmu_capable,
1535         .domain_alloc           = arm_smmu_domain_alloc,
1536         .domain_free            = arm_smmu_domain_free,
1537         .attach_dev             = arm_smmu_attach_dev,
1538         .map                    = arm_smmu_map,
1539         .unmap                  = arm_smmu_unmap,
1540         .map_sg                 = default_iommu_map_sg,
1541         .iova_to_phys           = arm_smmu_iova_to_phys,
1542         .add_device             = arm_smmu_add_device,
1543         .remove_device          = arm_smmu_remove_device,
1544         .device_group           = arm_smmu_device_group,
1545         .domain_get_attr        = arm_smmu_domain_get_attr,
1546         .domain_set_attr        = arm_smmu_domain_set_attr,
1547         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1548 };
1549 
1550 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1551 {
1552         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1553         void __iomem *cb_base;
1554         int i = 0;
1555         u32 reg, major;
1556 
1557         /* clear global FSR */
1558         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1559         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1560 
1561         /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1562         reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1563         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1564                 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1565                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1566         }
1567 
1568         /*
1569          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1570          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1571          * bit is only present in MMU-500r2 onwards.
1572          */
1573         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1574         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1575         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1576                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1577                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1578                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1579         }
1580 
1581         /* Make sure all context banks are disabled and clear CB_FSR  */
1582         for (i = 0; i < smmu->num_context_banks; ++i) {
1583                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1584                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1585                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1586                 /*
1587                  * Disable MMU-500's not-particularly-beneficial next-page
1588                  * prefetcher for the sake of errata #841119 and #826419.
1589                  */
1590                 if (smmu->model == ARM_MMU500) {
1591                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1592                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1593                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1594                 }
1595         }
1596 
1597         /* Invalidate the TLB, just in case */
1598         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1599         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1600 
1601         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1602 
1603         /* Enable fault reporting */
1604         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1605 
1606         /* Disable TLB broadcasting. */
1607         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1608 
1609         /* Enable client access, handling unmatched streams as appropriate */
1610         reg &= ~sCR0_CLIENTPD;
1611         if (disable_bypass)
1612                 reg |= sCR0_USFCFG;
1613         else
1614                 reg &= ~sCR0_USFCFG;
1615 
1616         /* Disable forced broadcasting */
1617         reg &= ~sCR0_FB;
1618 
1619         /* Don't upgrade barriers */
1620         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1621 
1622         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1623                 reg |= sCR0_VMID16EN;
1624 
1625         /* Push the button */
1626         __arm_smmu_tlb_sync(smmu);
1627         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1628 }
1629 
1630 static int arm_smmu_id_size_to_bits(int size)
1631 {
1632         switch (size) {
1633         case 0:
1634                 return 32;
1635         case 1:
1636                 return 36;
1637         case 2:
1638                 return 40;
1639         case 3:
1640                 return 42;
1641         case 4:
1642                 return 44;
1643         case 5:
1644         default:
1645                 return 48;
1646         }
1647 }
1648 
1649 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1650 {
1651         unsigned long size;
1652         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1653         u32 id;
1654         bool cttw_dt, cttw_reg;
1655 
1656         dev_notice(smmu->dev, "probing hardware configuration...\n");
1657         dev_notice(smmu->dev, "SMMUv%d with:\n",
1658                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1659 
1660         /* ID0 */
1661         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1662 
1663         /* Restrict available stages based on module parameter */
1664         if (force_stage == 1)
1665                 id &= ~(ID0_S2TS | ID0_NTS);
1666         else if (force_stage == 2)
1667                 id &= ~(ID0_S1TS | ID0_NTS);
1668 
1669         if (id & ID0_S1TS) {
1670                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1671                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1672         }
1673 
1674         if (id & ID0_S2TS) {
1675                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1676                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1677         }
1678 
1679         if (id & ID0_NTS) {
1680                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1681                 dev_notice(smmu->dev, "\tnested translation\n");
1682         }
1683 
1684         if (!(smmu->features &
1685                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1686                 dev_err(smmu->dev, "\tno translation support!\n");
1687                 return -ENODEV;
1688         }
1689 
1690         if ((id & ID0_S1TS) &&
1691                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1692                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1693                 dev_notice(smmu->dev, "\taddress translation ops\n");
1694         }
1695 
1696         /*
1697          * In order for DMA API calls to work properly, we must defer to what
1698          * the DT says about coherency, regardless of what the hardware claims.
1699          * Fortunately, this also opens up a workaround for systems where the
1700          * ID register value has ended up configured incorrectly.
1701          */
1702         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1703         cttw_reg = !!(id & ID0_CTTW);
1704         if (cttw_dt)
1705                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1706         if (cttw_dt || cttw_reg)
1707                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1708                            cttw_dt ? "" : "non-");
1709         if (cttw_dt != cttw_reg)
1710                 dev_notice(smmu->dev,
1711                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1712 
1713         if (id & ID0_SMS) {
1714                 u32 smr, sid, mask;
1715 
1716                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1717                 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1718                                            ID0_NUMSMRG_MASK;
1719                 if (smmu->num_mapping_groups == 0) {
1720                         dev_err(smmu->dev,
1721                                 "stream-matching supported, but no SMRs present!\n");
1722                         return -ENODEV;
1723                 }
1724 
1725                 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1726                 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1727                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1728                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1729 
1730                 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1731                 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1732                 if ((mask & sid) != sid) {
1733                         dev_err(smmu->dev,
1734                                 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1735                                 mask, sid);
1736                         return -ENODEV;
1737                 }
1738 
1739                 dev_notice(smmu->dev,
1740                            "\tstream matching with %u register groups, mask 0x%x",
1741                            smmu->num_mapping_groups, mask);
1742         } else {
1743                 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1744                                            ID0_NUMSIDB_MASK;
1745         }
1746 
1747         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1748                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1749                 if (!(id & ID0_PTFS_NO_AARCH32S))
1750                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1751         }
1752 
1753         /* ID1 */
1754         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1755         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1756 
1757         /* Check for size mismatch of SMMU address space from mapped region */
1758         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1759         size *= 2 << smmu->pgshift;
1760         if (smmu->size != size)
1761                 dev_warn(smmu->dev,
1762                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1763                         size, smmu->size);
1764 
1765         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1766         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1767         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1768                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1769                 return -ENODEV;
1770         }
1771         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1772                    smmu->num_context_banks, smmu->num_s2_context_banks);
1773         /*
1774          * Cavium CN88xx erratum #27704.
1775          * Ensure ASID and VMID allocation is unique across all SMMUs in
1776          * the system.
1777          */
1778         if (smmu->model == CAVIUM_SMMUV2) {
1779                 smmu->cavium_id_base =
1780                         atomic_add_return(smmu->num_context_banks,
1781                                           &cavium_smmu_context_count);
1782                 smmu->cavium_id_base -= smmu->num_context_banks;
1783         }
1784 
1785         /* ID2 */
1786         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1787         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1788         smmu->ipa_size = size;
1789 
1790         /* The output mask is also applied for bypass */
1791         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1792         smmu->pa_size = size;
1793 
1794         if (id & ID2_VMID16)
1795                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1796 
1797         /*
1798          * What the page table walker can address actually depends on which
1799          * descriptor format is in use, but since a) we don't know that yet,
1800          * and b) it can vary per context bank, this will have to do...
1801          */
1802         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1803                 dev_warn(smmu->dev,
1804                          "failed to set DMA mask for table walker\n");
1805 
1806         if (smmu->version < ARM_SMMU_V2) {
1807                 smmu->va_size = smmu->ipa_size;
1808                 if (smmu->version == ARM_SMMU_V1_64K)
1809                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1810         } else {
1811                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1812                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1813                 if (id & ID2_PTFS_4K)
1814                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1815                 if (id & ID2_PTFS_16K)
1816                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1817                 if (id & ID2_PTFS_64K)
1818                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1819         }
1820 
1821         /* Now we've corralled the various formats, what'll it do? */
1822         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1823                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1824         if (smmu->features &
1825             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1826                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1827         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1828                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1829         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1830                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1831 
1832         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1833                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1834         else
1835                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1836         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1837                    smmu->pgsize_bitmap);
1838 
1839 
1840         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1841                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1842                            smmu->va_size, smmu->ipa_size);
1843 
1844         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1845                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1846                            smmu->ipa_size, smmu->pa_size);
1847 
1848         return 0;
1849 }
1850 
1851 struct arm_smmu_match_data {
1852         enum arm_smmu_arch_version version;
1853         enum arm_smmu_implementation model;
1854 };
1855 
1856 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1857 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1858 
1859 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1860 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1861 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1862 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1863 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1864 
1865 static const struct of_device_id arm_smmu_of_match[] = {
1866         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1867         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1868         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1869         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1870         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1871         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1872         { },
1873 };
1874 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1875 
1876 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1877 {
1878         const struct of_device_id *of_id;
1879         const struct arm_smmu_match_data *data;
1880         struct resource *res;
1881         struct arm_smmu_device *smmu;
1882         struct device *dev = &pdev->dev;
1883         struct rb_node *node;
1884         struct of_phandle_iterator it;
1885         struct arm_smmu_phandle_args *masterspec;
1886         int num_irqs, i, err;
1887 
1888         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1889         if (!smmu) {
1890                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1891                 return -ENOMEM;
1892         }
1893         smmu->dev = dev;
1894 
1895         of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1896         data = of_id->data;
1897         smmu->version = data->version;
1898         smmu->model = data->model;
1899 
1900         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1901         smmu->base = devm_ioremap_resource(dev, res);
1902         if (IS_ERR(smmu->base))
1903                 return PTR_ERR(smmu->base);
1904         smmu->size = resource_size(res);
1905 
1906         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1907                                  &smmu->num_global_irqs)) {
1908                 dev_err(dev, "missing #global-interrupts property\n");
1909                 return -ENODEV;
1910         }
1911 
1912         num_irqs = 0;
1913         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1914                 num_irqs++;
1915                 if (num_irqs > smmu->num_global_irqs)
1916                         smmu->num_context_irqs++;
1917         }
1918 
1919         if (!smmu->num_context_irqs) {
1920                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1921                         num_irqs, smmu->num_global_irqs + 1);
1922                 return -ENODEV;
1923         }
1924 
1925         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1926                                   GFP_KERNEL);
1927         if (!smmu->irqs) {
1928                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1929                 return -ENOMEM;
1930         }
1931 
1932         for (i = 0; i < num_irqs; ++i) {
1933                 int irq = platform_get_irq(pdev, i);
1934 
1935                 if (irq < 0) {
1936                         dev_err(dev, "failed to get irq index %d\n", i);
1937                         return -ENODEV;
1938                 }
1939                 smmu->irqs[i] = irq;
1940         }
1941 
1942         err = arm_smmu_device_cfg_probe(smmu);
1943         if (err)
1944                 return err;
1945 
1946         i = 0;
1947         smmu->masters = RB_ROOT;
1948 
1949         err = -ENOMEM;
1950         /* No need to zero the memory for masterspec */
1951         masterspec = kmalloc(sizeof(*masterspec), GFP_KERNEL);
1952         if (!masterspec)
1953                 goto out_put_masters;
1954 
1955         of_for_each_phandle(&it, err, dev->of_node,
1956                             "mmu-masters", "#stream-id-cells", 0) {
1957                 int count = of_phandle_iterator_args(&it, masterspec->args,
1958                                                      MAX_MASTER_STREAMIDS);
1959                 masterspec->np          = of_node_get(it.node);
1960                 masterspec->args_count  = count;
1961 
1962                 err = register_smmu_master(smmu, dev, masterspec);
1963                 if (err) {
1964                         dev_err(dev, "failed to add master %s\n",
1965                                 masterspec->np->name);
1966                         kfree(masterspec);
1967                         goto out_put_masters;
1968                 }
1969 
1970                 i++;
1971         }
1972 
1973         dev_notice(dev, "registered %d master devices\n", i);
1974 
1975         kfree(masterspec);
1976 
1977         parse_driver_options(smmu);
1978 
1979         if (smmu->version == ARM_SMMU_V2 &&
1980             smmu->num_context_banks != smmu->num_context_irqs) {
1981                 dev_err(dev,
1982                         "found only %d context interrupt(s) but %d required\n",
1983                         smmu->num_context_irqs, smmu->num_context_banks);
1984                 err = -ENODEV;
1985                 goto out_put_masters;
1986         }
1987 
1988         for (i = 0; i < smmu->num_global_irqs; ++i) {
1989                 err = request_irq(smmu->irqs[i],
1990                                   arm_smmu_global_fault,
1991                                   IRQF_SHARED,
1992                                   "arm-smmu global fault",
1993                                   smmu);
1994                 if (err) {
1995                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
1996                                 i, smmu->irqs[i]);
1997                         goto out_free_irqs;
1998                 }
1999         }
2000 
2001         INIT_LIST_HEAD(&smmu->list);
2002         spin_lock(&arm_smmu_devices_lock);
2003         list_add(&smmu->list, &arm_smmu_devices);
2004         spin_unlock(&arm_smmu_devices_lock);
2005 
2006         arm_smmu_device_reset(smmu);
2007         return 0;
2008 
2009 out_free_irqs:
2010         while (i--)
2011                 free_irq(smmu->irqs[i], smmu);
2012 
2013 out_put_masters:
2014         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2015                 struct arm_smmu_master *master
2016                         = container_of(node, struct arm_smmu_master, node);
2017                 of_node_put(master->of_node);
2018         }
2019 
2020         return err;
2021 }
2022 
2023 static int arm_smmu_device_remove(struct platform_device *pdev)
2024 {
2025         int i;
2026         struct device *dev = &pdev->dev;
2027         struct arm_smmu_device *curr, *smmu = NULL;
2028         struct rb_node *node;
2029 
2030         spin_lock(&arm_smmu_devices_lock);
2031         list_for_each_entry(curr, &arm_smmu_devices, list) {
2032                 if (curr->dev == dev) {
2033                         smmu = curr;
2034                         list_del(&smmu->list);
2035                         break;
2036                 }
2037         }
2038         spin_unlock(&arm_smmu_devices_lock);
2039 
2040         if (!smmu)
2041                 return -ENODEV;
2042 
2043         for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2044                 struct arm_smmu_master *master
2045                         = container_of(node, struct arm_smmu_master, node);
2046                 of_node_put(master->of_node);
2047         }
2048 
2049         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2050                 dev_err(dev, "removing device with active domains!\n");
2051 
2052         for (i = 0; i < smmu->num_global_irqs; ++i)
2053                 free_irq(smmu->irqs[i], smmu);
2054 
2055         /* Turn the thing off */
2056         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2057         return 0;
2058 }
2059 
2060 static struct platform_driver arm_smmu_driver = {
2061         .driver = {
2062                 .name           = "arm-smmu",
2063                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2064         },
2065         .probe  = arm_smmu_device_dt_probe,
2066         .remove = arm_smmu_device_remove,
2067 };
2068 
2069 static int __init arm_smmu_init(void)
2070 {
2071         struct device_node *np;
2072         int ret;
2073 
2074         /*
2075          * Play nice with systems that don't have an ARM SMMU by checking that
2076          * an ARM SMMU exists in the system before proceeding with the driver
2077          * and IOMMU bus operation registration.
2078          */
2079         np = of_find_matching_node(NULL, arm_smmu_of_match);
2080         if (!np)
2081                 return 0;
2082 
2083         of_node_put(np);
2084 
2085         ret = platform_driver_register(&arm_smmu_driver);
2086         if (ret)
2087                 return ret;
2088 
2089         /* Oh, for a proper bus abstraction */
2090         if (!iommu_present(&platform_bus_type))
2091                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2092 
2093 #ifdef CONFIG_ARM_AMBA
2094         if (!iommu_present(&amba_bustype))
2095                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2096 #endif
2097 
2098 #ifdef CONFIG_PCI
2099         if (!iommu_present(&pci_bus_type))
2100                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2101 #endif
2102 
2103         return 0;
2104 }
2105 
2106 static void __exit arm_smmu_exit(void)
2107 {
2108         return platform_driver_unregister(&arm_smmu_driver);
2109 }
2110 
2111 subsys_initcall(arm_smmu_init);
2112 module_exit(arm_smmu_exit);
2113 
2114 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2115 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2116 MODULE_LICENSE("GPL v2");
2117 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us