blob: b493c99e17f74de338805167c3ddb104e3354b62 [file] [log] [blame]
Will Deacon45ae7cf2013-06-24 18:31:25 +01001/*
2 * IOMMU API for ARM architected SMMU implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 *
17 * Copyright (C) 2013 ARM Limited
18 *
19 * Author: Will Deacon <will.deacon@arm.com>
20 *
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
Will Deacon45ae7cf2013-06-24 18:31:25 +010026 * - Context fault reporting
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +030027 * - Extended Stream ID (16 bit)
Will Deacon45ae7cf2013-06-24 18:31:25 +010028 */
29
30#define pr_fmt(fmt) "arm-smmu: " fmt
31
Lorenzo Pieralisid6fcd3b2016-11-21 10:01:45 +000032#include <linux/acpi.h>
33#include <linux/acpi_iort.h>
Robin Murphy1f3d5ca2016-09-12 17:13:49 +010034#include <linux/atomic.h>
Will Deacon45ae7cf2013-06-24 18:31:25 +010035#include <linux/delay.h>
Robin Murphy9adb9592016-01-26 18:06:36 +000036#include <linux/dma-iommu.h>
Will Deacon45ae7cf2013-06-24 18:31:25 +010037#include <linux/dma-mapping.h>
38#include <linux/err.h>
39#include <linux/interrupt.h>
40#include <linux/io.h>
Robin Murphyf9a05f02016-04-13 18:13:01 +010041#include <linux/io-64-nonatomic-hi-lo.h>
Will Deacon45ae7cf2013-06-24 18:31:25 +010042#include <linux/iommu.h>
Mitchel Humpherys859a7322014-10-29 21:13:40 +000043#include <linux/iopoll.h>
Will Deacon45ae7cf2013-06-24 18:31:25 +010044#include <linux/module.h>
45#include <linux/of.h>
Robin Murphybae2c2d2015-07-29 19:46:05 +010046#include <linux/of_address.h>
Robin Murphyd6fc5d92016-09-12 17:13:52 +010047#include <linux/of_device.h>
Robin Murphyadfec2e2016-09-12 17:13:55 +010048#include <linux/of_iommu.h>
Will Deacona9a1b0b2014-05-01 18:05:08 +010049#include <linux/pci.h>
Will Deacon45ae7cf2013-06-24 18:31:25 +010050#include <linux/platform_device.h>
51#include <linux/slab.h>
52#include <linux/spinlock.h>
53
54#include <linux/amba/bus.h>
55
Will Deacon518f7132014-11-14 17:17:54 +000056#include "io-pgtable.h"
Will Deacon45ae7cf2013-06-24 18:31:25 +010057
Will Deacon45ae7cf2013-06-24 18:31:25 +010058/* Maximum number of context banks per SMMU */
59#define ARM_SMMU_MAX_CBS 128
60
Will Deacon45ae7cf2013-06-24 18:31:25 +010061/* SMMU global address space */
62#define ARM_SMMU_GR0(smmu) ((smmu)->base)
Will Deaconc757e852014-07-30 11:33:25 +010063#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
Will Deacon45ae7cf2013-06-24 18:31:25 +010064
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +000065/*
66 * SMMU global address space with conditional offset to access secure
67 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
68 * nsGFSYNR0: 0x450)
69 */
70#define ARM_SMMU_GR0_NS(smmu) \
71 ((smmu)->base + \
72 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
73 ? 0x400 : 0))
74
Robin Murphyf9a05f02016-04-13 18:13:01 +010075/*
76 * Some 64-bit registers only make sense to write atomically, but in such
77 * cases all the data relevant to AArch32 formats lies within the lower word,
78 * therefore this actually makes more sense than it might first appear.
79 */
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +010080#ifdef CONFIG_64BIT
Robin Murphyf9a05f02016-04-13 18:13:01 +010081#define smmu_write_atomic_lq writeq_relaxed
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +010082#else
Robin Murphyf9a05f02016-04-13 18:13:01 +010083#define smmu_write_atomic_lq writel_relaxed
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +010084#endif
85
Will Deacon45ae7cf2013-06-24 18:31:25 +010086/* Configuration registers */
87#define ARM_SMMU_GR0_sCR0 0x0
88#define sCR0_CLIENTPD (1 << 0)
89#define sCR0_GFRE (1 << 1)
90#define sCR0_GFIE (1 << 2)
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +030091#define sCR0_EXIDENABLE (1 << 3)
Will Deacon45ae7cf2013-06-24 18:31:25 +010092#define sCR0_GCFGFRE (1 << 4)
93#define sCR0_GCFGFIE (1 << 5)
94#define sCR0_USFCFG (1 << 10)
95#define sCR0_VMIDPNE (1 << 11)
96#define sCR0_PTM (1 << 12)
97#define sCR0_FB (1 << 13)
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -080098#define sCR0_VMID16EN (1 << 31)
Will Deacon45ae7cf2013-06-24 18:31:25 +010099#define sCR0_BSU_SHIFT 14
100#define sCR0_BSU_MASK 0x3
101
Peng Fan3ca37122016-05-03 21:50:30 +0800102/* Auxiliary Configuration register */
103#define ARM_SMMU_GR0_sACR 0x10
104
Will Deacon45ae7cf2013-06-24 18:31:25 +0100105/* Identification registers */
106#define ARM_SMMU_GR0_ID0 0x20
107#define ARM_SMMU_GR0_ID1 0x24
108#define ARM_SMMU_GR0_ID2 0x28
109#define ARM_SMMU_GR0_ID3 0x2c
110#define ARM_SMMU_GR0_ID4 0x30
111#define ARM_SMMU_GR0_ID5 0x34
112#define ARM_SMMU_GR0_ID6 0x38
113#define ARM_SMMU_GR0_ID7 0x3c
114#define ARM_SMMU_GR0_sGFSR 0x48
115#define ARM_SMMU_GR0_sGFSYNR0 0x50
116#define ARM_SMMU_GR0_sGFSYNR1 0x54
117#define ARM_SMMU_GR0_sGFSYNR2 0x58
Will Deacon45ae7cf2013-06-24 18:31:25 +0100118
119#define ID0_S1TS (1 << 30)
120#define ID0_S2TS (1 << 29)
121#define ID0_NTS (1 << 28)
122#define ID0_SMS (1 << 27)
Mitchel Humpherys859a7322014-10-29 21:13:40 +0000123#define ID0_ATOSNS (1 << 26)
Robin Murphy7602b872016-04-28 17:12:09 +0100124#define ID0_PTFS_NO_AARCH32 (1 << 25)
125#define ID0_PTFS_NO_AARCH32S (1 << 24)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100126#define ID0_CTTW (1 << 14)
127#define ID0_NUMIRPT_SHIFT 16
128#define ID0_NUMIRPT_MASK 0xff
Olav Haugan3c8766d2014-08-22 17:12:32 -0700129#define ID0_NUMSIDB_SHIFT 9
130#define ID0_NUMSIDB_MASK 0xf
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +0300131#define ID0_EXIDS (1 << 8)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100132#define ID0_NUMSMRG_SHIFT 0
133#define ID0_NUMSMRG_MASK 0xff
134
135#define ID1_PAGESIZE (1 << 31)
136#define ID1_NUMPAGENDXB_SHIFT 28
137#define ID1_NUMPAGENDXB_MASK 7
138#define ID1_NUMS2CB_SHIFT 16
139#define ID1_NUMS2CB_MASK 0xff
140#define ID1_NUMCB_SHIFT 0
141#define ID1_NUMCB_MASK 0xff
142
143#define ID2_OAS_SHIFT 4
144#define ID2_OAS_MASK 0xf
145#define ID2_IAS_SHIFT 0
146#define ID2_IAS_MASK 0xf
147#define ID2_UBS_SHIFT 8
148#define ID2_UBS_MASK 0xf
149#define ID2_PTFS_4K (1 << 12)
150#define ID2_PTFS_16K (1 << 13)
151#define ID2_PTFS_64K (1 << 14)
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800152#define ID2_VMID16 (1 << 15)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100153
Peng Fan3ca37122016-05-03 21:50:30 +0800154#define ID7_MAJOR_SHIFT 4
155#define ID7_MAJOR_MASK 0xf
Will Deacon45ae7cf2013-06-24 18:31:25 +0100156
Will Deacon45ae7cf2013-06-24 18:31:25 +0100157/* Global TLB invalidation */
Will Deacon45ae7cf2013-06-24 18:31:25 +0100158#define ARM_SMMU_GR0_TLBIVMID 0x64
159#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
160#define ARM_SMMU_GR0_TLBIALLH 0x6c
161#define ARM_SMMU_GR0_sTLBGSYNC 0x70
162#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
163#define sTLBGSTATUS_GSACTIVE (1 << 0)
164#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
165
166/* Stream mapping registers */
167#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
168#define SMR_VALID (1 << 31)
169#define SMR_MASK_SHIFT 16
Will Deacon45ae7cf2013-06-24 18:31:25 +0100170#define SMR_ID_SHIFT 0
Will Deacon45ae7cf2013-06-24 18:31:25 +0100171
172#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
173#define S2CR_CBNDX_SHIFT 0
174#define S2CR_CBNDX_MASK 0xff
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +0300175#define S2CR_EXIDVALID (1 << 10)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100176#define S2CR_TYPE_SHIFT 16
177#define S2CR_TYPE_MASK 0x3
Robin Murphy8e8b2032016-09-12 17:13:50 +0100178enum arm_smmu_s2cr_type {
179 S2CR_TYPE_TRANS,
180 S2CR_TYPE_BYPASS,
181 S2CR_TYPE_FAULT,
182};
Will Deacon45ae7cf2013-06-24 18:31:25 +0100183
Robin Murphyd3461802016-01-26 18:06:34 +0000184#define S2CR_PRIVCFG_SHIFT 24
Robin Murphy8e8b2032016-09-12 17:13:50 +0100185#define S2CR_PRIVCFG_MASK 0x3
186enum arm_smmu_s2cr_privcfg {
187 S2CR_PRIVCFG_DEFAULT,
188 S2CR_PRIVCFG_DIPAN,
189 S2CR_PRIVCFG_UNPRIV,
190 S2CR_PRIVCFG_PRIV,
191};
Robin Murphyd3461802016-01-26 18:06:34 +0000192
Will Deacon45ae7cf2013-06-24 18:31:25 +0100193/* Context bank attribute registers */
194#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
195#define CBAR_VMID_SHIFT 0
196#define CBAR_VMID_MASK 0xff
Will Deacon57ca90f2014-02-06 14:59:05 +0000197#define CBAR_S1_BPSHCFG_SHIFT 8
198#define CBAR_S1_BPSHCFG_MASK 3
199#define CBAR_S1_BPSHCFG_NSH 3
Will Deacon45ae7cf2013-06-24 18:31:25 +0100200#define CBAR_S1_MEMATTR_SHIFT 12
201#define CBAR_S1_MEMATTR_MASK 0xf
202#define CBAR_S1_MEMATTR_WB 0xf
203#define CBAR_TYPE_SHIFT 16
204#define CBAR_TYPE_MASK 0x3
205#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
206#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
207#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
208#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
209#define CBAR_IRPTNDX_SHIFT 24
210#define CBAR_IRPTNDX_MASK 0xff
211
212#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
213#define CBA2R_RW64_32BIT (0 << 0)
214#define CBA2R_RW64_64BIT (1 << 0)
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800215#define CBA2R_VMID_SHIFT 16
216#define CBA2R_VMID_MASK 0xffff
Will Deacon45ae7cf2013-06-24 18:31:25 +0100217
218/* Translation context bank */
219#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
Will Deaconc757e852014-07-30 11:33:25 +0100220#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
Will Deacon45ae7cf2013-06-24 18:31:25 +0100221
222#define ARM_SMMU_CB_SCTLR 0x0
Robin Murphyf0cfffc2016-04-13 18:12:59 +0100223#define ARM_SMMU_CB_ACTLR 0x4
Will Deacon45ae7cf2013-06-24 18:31:25 +0100224#define ARM_SMMU_CB_RESUME 0x8
225#define ARM_SMMU_CB_TTBCR2 0x10
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +0100226#define ARM_SMMU_CB_TTBR0 0x20
227#define ARM_SMMU_CB_TTBR1 0x28
Will Deacon45ae7cf2013-06-24 18:31:25 +0100228#define ARM_SMMU_CB_TTBCR 0x30
Robin Murphy60705292016-08-11 17:44:06 +0100229#define ARM_SMMU_CB_CONTEXTIDR 0x34
Will Deacon45ae7cf2013-06-24 18:31:25 +0100230#define ARM_SMMU_CB_S1_MAIR0 0x38
Will Deacon518f7132014-11-14 17:17:54 +0000231#define ARM_SMMU_CB_S1_MAIR1 0x3c
Robin Murphyf9a05f02016-04-13 18:13:01 +0100232#define ARM_SMMU_CB_PAR 0x50
Will Deacon45ae7cf2013-06-24 18:31:25 +0100233#define ARM_SMMU_CB_FSR 0x58
Robin Murphyf9a05f02016-04-13 18:13:01 +0100234#define ARM_SMMU_CB_FAR 0x60
Will Deacon45ae7cf2013-06-24 18:31:25 +0100235#define ARM_SMMU_CB_FSYNR0 0x68
Will Deacon518f7132014-11-14 17:17:54 +0000236#define ARM_SMMU_CB_S1_TLBIVA 0x600
Will Deacon1463fe42013-07-31 19:21:27 +0100237#define ARM_SMMU_CB_S1_TLBIASID 0x610
Will Deacon518f7132014-11-14 17:17:54 +0000238#define ARM_SMMU_CB_S1_TLBIVAL 0x620
239#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
240#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
Robin Murphy661d9622015-05-27 17:09:34 +0100241#define ARM_SMMU_CB_ATS1PR 0x800
Mitchel Humpherys859a7322014-10-29 21:13:40 +0000242#define ARM_SMMU_CB_ATSR 0x8f0
Will Deacon45ae7cf2013-06-24 18:31:25 +0100243
244#define SCTLR_S1_ASIDPNE (1 << 12)
245#define SCTLR_CFCFG (1 << 7)
246#define SCTLR_CFIE (1 << 6)
247#define SCTLR_CFRE (1 << 5)
248#define SCTLR_E (1 << 4)
249#define SCTLR_AFE (1 << 2)
250#define SCTLR_TRE (1 << 1)
251#define SCTLR_M (1 << 0)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100252
Robin Murphyf0cfffc2016-04-13 18:12:59 +0100253#define ARM_MMU500_ACTLR_CPRE (1 << 1)
254
Peng Fan3ca37122016-05-03 21:50:30 +0800255#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
Nipun Gupta6eb18d42016-11-04 15:25:23 +0530256#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
Peng Fan3ca37122016-05-03 21:50:30 +0800257
Mitchel Humpherys859a7322014-10-29 21:13:40 +0000258#define CB_PAR_F (1 << 0)
259
260#define ATSR_ACTIVE (1 << 0)
261
Will Deacon45ae7cf2013-06-24 18:31:25 +0100262#define RESUME_RETRY (0 << 0)
263#define RESUME_TERMINATE (1 << 0)
264
Will Deacon45ae7cf2013-06-24 18:31:25 +0100265#define TTBCR2_SEP_SHIFT 15
Will Deacon5dc56162015-05-08 17:44:22 +0100266#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
Tomasz Nowicki3677a642017-01-16 08:16:07 +0100267#define TTBCR2_AS (1 << 4)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100268
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +0100269#define TTBRn_ASID_SHIFT 48
Will Deacon45ae7cf2013-06-24 18:31:25 +0100270
271#define FSR_MULTI (1 << 31)
272#define FSR_SS (1 << 30)
273#define FSR_UUT (1 << 8)
274#define FSR_ASF (1 << 7)
275#define FSR_TLBLKF (1 << 6)
276#define FSR_TLBMCF (1 << 5)
277#define FSR_EF (1 << 4)
278#define FSR_PF (1 << 3)
279#define FSR_AFF (1 << 2)
280#define FSR_TF (1 << 1)
281
Mitchel Humpherys29073202014-07-08 09:52:18 -0700282#define FSR_IGN (FSR_AFF | FSR_ASF | \
283 FSR_TLBMCF | FSR_TLBLKF)
284#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
Will Deaconadaba322013-07-31 19:21:26 +0100285 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100286
287#define FSYNR0_WNR (1 << 4)
288
Eric Augerf3ebee82017-01-19 20:57:55 +0000289#define MSI_IOVA_BASE 0x8000000
290#define MSI_IOVA_LENGTH 0x100000
291
Will Deacon4cf740b2014-07-14 19:47:39 +0100292static int force_stage;
Robin Murphy25a1c962016-02-10 14:25:33 +0000293module_param(force_stage, int, S_IRUGO);
Will Deacon4cf740b2014-07-14 19:47:39 +0100294MODULE_PARM_DESC(force_stage,
295 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
Robin Murphy25a1c962016-02-10 14:25:33 +0000296static bool disable_bypass;
297module_param(disable_bypass, bool, S_IRUGO);
298MODULE_PARM_DESC(disable_bypass,
299 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
Will Deacon4cf740b2014-07-14 19:47:39 +0100300
Robin Murphy09360402014-08-28 17:51:59 +0100301enum arm_smmu_arch_version {
Robin Murphyb7862e32016-04-13 18:13:03 +0100302 ARM_SMMU_V1,
303 ARM_SMMU_V1_64K,
Robin Murphy09360402014-08-28 17:51:59 +0100304 ARM_SMMU_V2,
305};
306
Robin Murphy67b65a32016-04-13 18:12:57 +0100307enum arm_smmu_implementation {
308 GENERIC_SMMU,
Robin Murphyf0cfffc2016-04-13 18:12:59 +0100309 ARM_MMU500,
Robin Murphye086d912016-04-13 18:12:58 +0100310 CAVIUM_SMMUV2,
Robin Murphy67b65a32016-04-13 18:12:57 +0100311};
312
Robin Murphy8e8b2032016-09-12 17:13:50 +0100313struct arm_smmu_s2cr {
Robin Murphy588888a2016-09-12 17:13:54 +0100314 struct iommu_group *group;
315 int count;
Robin Murphy8e8b2032016-09-12 17:13:50 +0100316 enum arm_smmu_s2cr_type type;
317 enum arm_smmu_s2cr_privcfg privcfg;
318 u8 cbndx;
319};
320
321#define s2cr_init_val (struct arm_smmu_s2cr){ \
322 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
323}
324
Will Deacon45ae7cf2013-06-24 18:31:25 +0100325struct arm_smmu_smr {
Will Deacon45ae7cf2013-06-24 18:31:25 +0100326 u16 mask;
327 u16 id;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +0100328 bool valid;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100329};
330
Will Deacona9a1b0b2014-05-01 18:05:08 +0100331struct arm_smmu_master_cfg {
Robin Murphyf80cd882016-09-14 15:21:39 +0100332 struct arm_smmu_device *smmu;
Robin Murphyadfec2e2016-09-12 17:13:55 +0100333 s16 smendx[];
Will Deacon45ae7cf2013-06-24 18:31:25 +0100334};
Robin Murphy1f3d5ca2016-09-12 17:13:49 +0100335#define INVALID_SMENDX -1
Robin Murphyadfec2e2016-09-12 17:13:55 +0100336#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
337#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
Robin Murphy8c82d6e2016-11-07 18:25:09 +0000338#define fwspec_smendx(fw, i) \
339 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
Robin Murphyadfec2e2016-09-12 17:13:55 +0100340#define for_each_cfg_sme(fw, i, idx) \
Robin Murphy8c82d6e2016-11-07 18:25:09 +0000341 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100342
343struct arm_smmu_device {
344 struct device *dev;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100345
346 void __iomem *base;
347 unsigned long size;
Will Deaconc757e852014-07-30 11:33:25 +0100348 unsigned long pgshift;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100349
350#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
351#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
352#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
353#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
354#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
Mitchel Humpherys859a7322014-10-29 21:13:40 +0000355#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800356#define ARM_SMMU_FEAT_VMID16 (1 << 6)
Robin Murphy7602b872016-04-28 17:12:09 +0100357#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
358#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
359#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
360#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
361#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +0300362#define ARM_SMMU_FEAT_EXIDS (1 << 12)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100363 u32 features;
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000364
365#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
366 u32 options;
Robin Murphy09360402014-08-28 17:51:59 +0100367 enum arm_smmu_arch_version version;
Robin Murphy67b65a32016-04-13 18:12:57 +0100368 enum arm_smmu_implementation model;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100369
370 u32 num_context_banks;
371 u32 num_s2_context_banks;
372 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
373 atomic_t irptndx;
374
375 u32 num_mapping_groups;
Robin Murphy21174242016-09-12 17:13:48 +0100376 u16 streamid_mask;
377 u16 smr_mask_mask;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +0100378 struct arm_smmu_smr *smrs;
Robin Murphy8e8b2032016-09-12 17:13:50 +0100379 struct arm_smmu_s2cr *s2crs;
Robin Murphy588888a2016-09-12 17:13:54 +0100380 struct mutex stream_map_mutex;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100381
Will Deacon518f7132014-11-14 17:17:54 +0000382 unsigned long va_size;
383 unsigned long ipa_size;
384 unsigned long pa_size;
Robin Murphyd5466352016-05-09 17:20:09 +0100385 unsigned long pgsize_bitmap;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100386
387 u32 num_global_irqs;
388 u32 num_context_irqs;
389 unsigned int *irqs;
390
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800391 u32 cavium_id_base; /* Specific to Cavium */
Joerg Roedel9648cbc2017-02-01 18:11:36 +0100392
393 /* IOMMU core code handle */
394 struct iommu_device iommu;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100395};
396
Robin Murphy7602b872016-04-28 17:12:09 +0100397enum arm_smmu_context_fmt {
398 ARM_SMMU_CTX_FMT_NONE,
399 ARM_SMMU_CTX_FMT_AARCH64,
400 ARM_SMMU_CTX_FMT_AARCH32_L,
401 ARM_SMMU_CTX_FMT_AARCH32_S,
Will Deacon45ae7cf2013-06-24 18:31:25 +0100402};
403
404struct arm_smmu_cfg {
Will Deacon45ae7cf2013-06-24 18:31:25 +0100405 u8 cbndx;
406 u8 irptndx;
407 u32 cbar;
Robin Murphy7602b872016-04-28 17:12:09 +0100408 enum arm_smmu_context_fmt fmt;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100409};
Dan Carpenterfaea13b72013-08-21 09:33:30 +0100410#define INVALID_IRPTNDX 0xff
Will Deacon45ae7cf2013-06-24 18:31:25 +0100411
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800412#define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
413#define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
Will Deaconecfadb62013-07-31 19:21:28 +0100414
Will Deaconc752ce42014-06-25 22:46:31 +0100415enum arm_smmu_domain_stage {
416 ARM_SMMU_DOMAIN_S1 = 0,
417 ARM_SMMU_DOMAIN_S2,
418 ARM_SMMU_DOMAIN_NESTED,
419};
420
Will Deacon45ae7cf2013-06-24 18:31:25 +0100421struct arm_smmu_domain {
Will Deacon44680ee2014-06-25 11:29:12 +0100422 struct arm_smmu_device *smmu;
Will Deacon518f7132014-11-14 17:17:54 +0000423 struct io_pgtable_ops *pgtbl_ops;
424 spinlock_t pgtbl_lock;
Will Deacon44680ee2014-06-25 11:29:12 +0100425 struct arm_smmu_cfg cfg;
Will Deaconc752ce42014-06-25 22:46:31 +0100426 enum arm_smmu_domain_stage stage;
Will Deacon518f7132014-11-14 17:17:54 +0000427 struct mutex init_mutex; /* Protects smmu pointer */
Joerg Roedel1d672632015-03-26 13:43:10 +0100428 struct iommu_domain domain;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100429};
430
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000431struct arm_smmu_option_prop {
432 u32 opt;
433 const char *prop;
434};
435
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800436static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
437
Robin Murphy021bb842016-09-14 15:26:46 +0100438static bool using_legacy_binding, using_generic_binding;
439
Mitchel Humpherys29073202014-07-08 09:52:18 -0700440static struct arm_smmu_option_prop arm_smmu_options[] = {
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000441 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
442 { 0, NULL},
443};
444
Joerg Roedel1d672632015-03-26 13:43:10 +0100445static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
446{
447 return container_of(dom, struct arm_smmu_domain, domain);
448}
449
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000450static void parse_driver_options(struct arm_smmu_device *smmu)
451{
452 int i = 0;
Mitchel Humpherys29073202014-07-08 09:52:18 -0700453
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000454 do {
455 if (of_property_read_bool(smmu->dev->of_node,
456 arm_smmu_options[i].prop)) {
457 smmu->options |= arm_smmu_options[i].opt;
458 dev_notice(smmu->dev, "option %s\n",
459 arm_smmu_options[i].prop);
460 }
461 } while (arm_smmu_options[++i].opt);
462}
463
Will Deacon8f68f8e2014-07-15 11:27:08 +0100464static struct device_node *dev_get_dev_node(struct device *dev)
Will Deacona9a1b0b2014-05-01 18:05:08 +0100465{
466 if (dev_is_pci(dev)) {
467 struct pci_bus *bus = to_pci_dev(dev)->bus;
Mitchel Humpherys29073202014-07-08 09:52:18 -0700468
Will Deacona9a1b0b2014-05-01 18:05:08 +0100469 while (!pci_is_root_bus(bus))
470 bus = bus->parent;
Robin Murphyf80cd882016-09-14 15:21:39 +0100471 return of_node_get(bus->bridge->parent->of_node);
Will Deacona9a1b0b2014-05-01 18:05:08 +0100472 }
473
Robin Murphyf80cd882016-09-14 15:21:39 +0100474 return of_node_get(dev->of_node);
Will Deacona9a1b0b2014-05-01 18:05:08 +0100475}
476
Robin Murphyf80cd882016-09-14 15:21:39 +0100477static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100478{
Robin Murphyf80cd882016-09-14 15:21:39 +0100479 *((__be32 *)data) = cpu_to_be32(alias);
480 return 0; /* Continue walking */
Will Deacon45ae7cf2013-06-24 18:31:25 +0100481}
482
Robin Murphyf80cd882016-09-14 15:21:39 +0100483static int __find_legacy_master_phandle(struct device *dev, void *data)
Will Deacona9a1b0b2014-05-01 18:05:08 +0100484{
Robin Murphyf80cd882016-09-14 15:21:39 +0100485 struct of_phandle_iterator *it = *(void **)data;
486 struct device_node *np = it->node;
487 int err;
Will Deacona9a1b0b2014-05-01 18:05:08 +0100488
Robin Murphyf80cd882016-09-14 15:21:39 +0100489 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
490 "#stream-id-cells", 0)
491 if (it->node == np) {
492 *(void **)data = dev;
493 return 1;
Olav Haugan3c8766d2014-08-22 17:12:32 -0700494 }
Robin Murphyf80cd882016-09-14 15:21:39 +0100495 it->node = np;
496 return err == -ENOENT ? 0 : err;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100497}
498
Robin Murphyd6fc5d92016-09-12 17:13:52 +0100499static struct platform_driver arm_smmu_driver;
Robin Murphyadfec2e2016-09-12 17:13:55 +0100500static struct iommu_ops arm_smmu_ops;
Robin Murphyd6fc5d92016-09-12 17:13:52 +0100501
Robin Murphyadfec2e2016-09-12 17:13:55 +0100502static int arm_smmu_register_legacy_master(struct device *dev,
503 struct arm_smmu_device **smmu)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100504{
Robin Murphyadfec2e2016-09-12 17:13:55 +0100505 struct device *smmu_dev;
Robin Murphyf80cd882016-09-14 15:21:39 +0100506 struct device_node *np;
507 struct of_phandle_iterator it;
508 void *data = &it;
Robin Murphyadfec2e2016-09-12 17:13:55 +0100509 u32 *sids;
Robin Murphyf80cd882016-09-14 15:21:39 +0100510 __be32 pci_sid;
511 int err;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100512
Robin Murphyf80cd882016-09-14 15:21:39 +0100513 np = dev_get_dev_node(dev);
514 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
515 of_node_put(np);
516 return -ENODEV;
517 }
518
519 it.node = np;
Robin Murphyd6fc5d92016-09-12 17:13:52 +0100520 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
521 __find_legacy_master_phandle);
Robin Murphyadfec2e2016-09-12 17:13:55 +0100522 smmu_dev = data;
Robin Murphyf80cd882016-09-14 15:21:39 +0100523 of_node_put(np);
524 if (err == 0)
525 return -ENODEV;
526 if (err < 0)
527 return err;
Will Deacon44680ee2014-06-25 11:29:12 +0100528
Robin Murphyf80cd882016-09-14 15:21:39 +0100529 if (dev_is_pci(dev)) {
530 /* "mmu-masters" assumes Stream ID == Requester ID */
531 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
532 &pci_sid);
533 it.cur = &pci_sid;
534 it.cur_count = 1;
535 }
536
Robin Murphyadfec2e2016-09-12 17:13:55 +0100537 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
538 &arm_smmu_ops);
539 if (err)
540 return err;
541
542 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
543 if (!sids)
Robin Murphyf80cd882016-09-14 15:21:39 +0100544 return -ENOMEM;
545
Robin Murphyadfec2e2016-09-12 17:13:55 +0100546 *smmu = dev_get_drvdata(smmu_dev);
547 of_phandle_iterator_args(&it, sids, it.cur_count);
548 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
549 kfree(sids);
550 return err;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100551}
552
553static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
554{
555 int idx;
556
557 do {
558 idx = find_next_zero_bit(map, end, start);
559 if (idx == end)
560 return -ENOSPC;
561 } while (test_and_set_bit(idx, map));
562
563 return idx;
564}
565
566static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
567{
568 clear_bit(idx, map);
569}
570
571/* Wait for any pending TLB invalidations to complete */
Will Deacon518f7132014-11-14 17:17:54 +0000572static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100573{
574 int count = 0;
575 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
576
577 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
578 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
579 & sTLBGSTATUS_GSACTIVE) {
580 cpu_relax();
581 if (++count == TLB_LOOP_TIMEOUT) {
582 dev_err_ratelimited(smmu->dev,
583 "TLB sync timed out -- SMMU may be deadlocked\n");
584 return;
585 }
586 udelay(1);
587 }
588}
589
Will Deacon518f7132014-11-14 17:17:54 +0000590static void arm_smmu_tlb_sync(void *cookie)
Will Deacon1463fe42013-07-31 19:21:27 +0100591{
Will Deacon518f7132014-11-14 17:17:54 +0000592 struct arm_smmu_domain *smmu_domain = cookie;
593 __arm_smmu_tlb_sync(smmu_domain->smmu);
594}
595
596static void arm_smmu_tlb_inv_context(void *cookie)
597{
598 struct arm_smmu_domain *smmu_domain = cookie;
Will Deacon44680ee2014-06-25 11:29:12 +0100599 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
600 struct arm_smmu_device *smmu = smmu_domain->smmu;
Will Deacon1463fe42013-07-31 19:21:27 +0100601 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
Will Deacon518f7132014-11-14 17:17:54 +0000602 void __iomem *base;
Will Deacon1463fe42013-07-31 19:21:27 +0100603
604 if (stage1) {
605 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800606 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
Will Deaconecfadb62013-07-31 19:21:28 +0100607 base + ARM_SMMU_CB_S1_TLBIASID);
Will Deacon1463fe42013-07-31 19:21:27 +0100608 } else {
609 base = ARM_SMMU_GR0(smmu);
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800610 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
Will Deaconecfadb62013-07-31 19:21:28 +0100611 base + ARM_SMMU_GR0_TLBIVMID);
Will Deacon1463fe42013-07-31 19:21:27 +0100612 }
613
Will Deacon518f7132014-11-14 17:17:54 +0000614 __arm_smmu_tlb_sync(smmu);
Will Deacon1463fe42013-07-31 19:21:27 +0100615}
616
Will Deacon518f7132014-11-14 17:17:54 +0000617static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
Robin Murphy06c610e2015-12-07 18:18:53 +0000618 size_t granule, bool leaf, void *cookie)
Will Deacon518f7132014-11-14 17:17:54 +0000619{
620 struct arm_smmu_domain *smmu_domain = cookie;
621 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
622 struct arm_smmu_device *smmu = smmu_domain->smmu;
623 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
624 void __iomem *reg;
625
626 if (stage1) {
627 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
628 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
629
Robin Murphy7602b872016-04-28 17:12:09 +0100630 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
Will Deacon518f7132014-11-14 17:17:54 +0000631 iova &= ~12UL;
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800632 iova |= ARM_SMMU_CB_ASID(smmu, cfg);
Robin Murphy75df1382015-12-07 18:18:52 +0000633 do {
634 writel_relaxed(iova, reg);
635 iova += granule;
636 } while (size -= granule);
Will Deacon518f7132014-11-14 17:17:54 +0000637 } else {
638 iova >>= 12;
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800639 iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
Robin Murphy75df1382015-12-07 18:18:52 +0000640 do {
641 writeq_relaxed(iova, reg);
642 iova += granule >> 12;
643 } while (size -= granule);
Will Deacon518f7132014-11-14 17:17:54 +0000644 }
Will Deacon518f7132014-11-14 17:17:54 +0000645 } else if (smmu->version == ARM_SMMU_V2) {
646 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
647 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
648 ARM_SMMU_CB_S2_TLBIIPAS2;
Robin Murphy75df1382015-12-07 18:18:52 +0000649 iova >>= 12;
650 do {
Robin Murphyf9a05f02016-04-13 18:13:01 +0100651 smmu_write_atomic_lq(iova, reg);
Robin Murphy75df1382015-12-07 18:18:52 +0000652 iova += granule >> 12;
653 } while (size -= granule);
Will Deacon518f7132014-11-14 17:17:54 +0000654 } else {
655 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800656 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
Will Deacon518f7132014-11-14 17:17:54 +0000657 }
658}
659
Bhumika Goyal5896f3a2016-10-25 23:36:10 +0530660static const struct iommu_gather_ops arm_smmu_gather_ops = {
Will Deacon518f7132014-11-14 17:17:54 +0000661 .tlb_flush_all = arm_smmu_tlb_inv_context,
662 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
663 .tlb_sync = arm_smmu_tlb_sync,
Will Deacon518f7132014-11-14 17:17:54 +0000664};
665
Will Deacon45ae7cf2013-06-24 18:31:25 +0100666static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
667{
Will Deacon3714ce1d2016-08-05 19:49:45 +0100668 u32 fsr, fsynr;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100669 unsigned long iova;
670 struct iommu_domain *domain = dev;
Joerg Roedel1d672632015-03-26 13:43:10 +0100671 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon44680ee2014-06-25 11:29:12 +0100672 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
673 struct arm_smmu_device *smmu = smmu_domain->smmu;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100674 void __iomem *cb_base;
675
Will Deacon44680ee2014-06-25 11:29:12 +0100676 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100677 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
678
679 if (!(fsr & FSR_FAULT))
680 return IRQ_NONE;
681
Will Deacon45ae7cf2013-06-24 18:31:25 +0100682 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
Robin Murphyf9a05f02016-04-13 18:13:01 +0100683 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100684
Will Deacon3714ce1d2016-08-05 19:49:45 +0100685 dev_err_ratelimited(smmu->dev,
686 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
687 fsr, iova, fsynr, cfg->cbndx);
688
Will Deacon45ae7cf2013-06-24 18:31:25 +0100689 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
Will Deacon3714ce1d2016-08-05 19:49:45 +0100690 return IRQ_HANDLED;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100691}
692
693static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
694{
695 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
696 struct arm_smmu_device *smmu = dev;
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000697 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100698
699 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
700 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
701 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
702 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
703
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +0000704 if (!gfsr)
705 return IRQ_NONE;
706
Will Deacon45ae7cf2013-06-24 18:31:25 +0100707 dev_err_ratelimited(smmu->dev,
708 "Unexpected global fault, this could be serious\n");
709 dev_err_ratelimited(smmu->dev,
710 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
711 gfsr, gfsynr0, gfsynr1, gfsynr2);
712
713 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
Will Deaconadaba322013-07-31 19:21:26 +0100714 return IRQ_HANDLED;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100715}
716
Will Deacon518f7132014-11-14 17:17:54 +0000717static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
718 struct io_pgtable_cfg *pgtbl_cfg)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100719{
Robin Murphy60705292016-08-11 17:44:06 +0100720 u32 reg, reg2;
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +0100721 u64 reg64;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100722 bool stage1;
Will Deacon44680ee2014-06-25 11:29:12 +0100723 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
724 struct arm_smmu_device *smmu = smmu_domain->smmu;
Will Deaconc88ae5d2015-10-13 17:53:24 +0100725 void __iomem *cb_base, *gr1_base;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100726
Will Deacon45ae7cf2013-06-24 18:31:25 +0100727 gr1_base = ARM_SMMU_GR1(smmu);
Will Deacon44680ee2014-06-25 11:29:12 +0100728 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
729 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100730
Will Deacon4a1c93c2015-03-04 12:21:03 +0000731 if (smmu->version > ARM_SMMU_V1) {
Robin Murphy7602b872016-04-28 17:12:09 +0100732 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
733 reg = CBA2R_RW64_64BIT;
734 else
735 reg = CBA2R_RW64_32BIT;
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800736 /* 16-bit VMIDs live in CBA2R */
737 if (smmu->features & ARM_SMMU_FEAT_VMID16)
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800738 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800739
Will Deacon4a1c93c2015-03-04 12:21:03 +0000740 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
741 }
742
Will Deacon45ae7cf2013-06-24 18:31:25 +0100743 /* CBAR */
Will Deacon44680ee2014-06-25 11:29:12 +0100744 reg = cfg->cbar;
Robin Murphyb7862e32016-04-13 18:13:03 +0100745 if (smmu->version < ARM_SMMU_V2)
Mitchel Humpherys29073202014-07-08 09:52:18 -0700746 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100747
Will Deacon57ca90f2014-02-06 14:59:05 +0000748 /*
749 * Use the weakest shareability/memory types, so they are
750 * overridden by the ttbcr/pte.
751 */
752 if (stage1) {
753 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
754 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -0800755 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
756 /* 8-bit VMIDs live in CBAR */
Tirumalesh Chalamarla1bd37a62016-03-04 13:56:09 -0800757 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
Will Deacon57ca90f2014-02-06 14:59:05 +0000758 }
Will Deacon44680ee2014-06-25 11:29:12 +0100759 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
Will Deacon45ae7cf2013-06-24 18:31:25 +0100760
Will Deacon518f7132014-11-14 17:17:54 +0000761 /* TTBRs */
762 if (stage1) {
Robin Murphy60705292016-08-11 17:44:06 +0100763 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100764
Robin Murphy60705292016-08-11 17:44:06 +0100765 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
766 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
767 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
768 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
769 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
770 writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
771 } else {
772 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
773 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
774 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
775 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
776 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
777 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
778 }
Will Deacon518f7132014-11-14 17:17:54 +0000779 } else {
Tirumalesh Chalamarla668b4ad2015-08-19 00:40:30 +0100780 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
Robin Murphyf9a05f02016-04-13 18:13:01 +0100781 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
Will Deacon518f7132014-11-14 17:17:54 +0000782 }
Will Deacon45ae7cf2013-06-24 18:31:25 +0100783
Will Deacon518f7132014-11-14 17:17:54 +0000784 /* TTBCR */
785 if (stage1) {
Robin Murphy60705292016-08-11 17:44:06 +0100786 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
787 reg = pgtbl_cfg->arm_v7s_cfg.tcr;
788 reg2 = 0;
789 } else {
790 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
791 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
792 reg2 |= TTBCR2_SEP_UPSTREAM;
Tomasz Nowicki3677a642017-01-16 08:16:07 +0100793 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
794 reg2 |= TTBCR2_AS;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100795 }
Robin Murphy60705292016-08-11 17:44:06 +0100796 if (smmu->version > ARM_SMMU_V1)
797 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100798 } else {
Will Deacon518f7132014-11-14 17:17:54 +0000799 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100800 }
Robin Murphy60705292016-08-11 17:44:06 +0100801 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100802
Will Deacon518f7132014-11-14 17:17:54 +0000803 /* MAIRs (stage-1 only) */
Will Deacon45ae7cf2013-06-24 18:31:25 +0100804 if (stage1) {
Robin Murphy60705292016-08-11 17:44:06 +0100805 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
806 reg = pgtbl_cfg->arm_v7s_cfg.prrr;
807 reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
808 } else {
809 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
810 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
811 }
Will Deacon45ae7cf2013-06-24 18:31:25 +0100812 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
Robin Murphy60705292016-08-11 17:44:06 +0100813 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100814 }
815
Will Deacon45ae7cf2013-06-24 18:31:25 +0100816 /* SCTLR */
Robin Murphy60705292016-08-11 17:44:06 +0100817 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100818 if (stage1)
819 reg |= SCTLR_S1_ASIDPNE;
820#ifdef __BIG_ENDIAN
821 reg |= SCTLR_E;
822#endif
Will Deacon25724842013-08-21 13:49:53 +0100823 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100824}
825
826static int arm_smmu_init_domain_context(struct iommu_domain *domain,
Will Deacon44680ee2014-06-25 11:29:12 +0100827 struct arm_smmu_device *smmu)
Will Deacon45ae7cf2013-06-24 18:31:25 +0100828{
Mitchel Humpherysa18037b2014-07-30 18:58:13 +0100829 int irq, start, ret = 0;
Will Deacon518f7132014-11-14 17:17:54 +0000830 unsigned long ias, oas;
831 struct io_pgtable_ops *pgtbl_ops;
832 struct io_pgtable_cfg pgtbl_cfg;
833 enum io_pgtable_fmt fmt;
Joerg Roedel1d672632015-03-26 13:43:10 +0100834 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon44680ee2014-06-25 11:29:12 +0100835 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100836
Will Deacon518f7132014-11-14 17:17:54 +0000837 mutex_lock(&smmu_domain->init_mutex);
Mitchel Humpherysa18037b2014-07-30 18:58:13 +0100838 if (smmu_domain->smmu)
839 goto out_unlock;
840
Will Deaconc752ce42014-06-25 22:46:31 +0100841 /*
842 * Mapping the requested stage onto what we support is surprisingly
843 * complicated, mainly because the spec allows S1+S2 SMMUs without
844 * support for nested translation. That means we end up with the
845 * following table:
846 *
847 * Requested Supported Actual
848 * S1 N S1
849 * S1 S1+S2 S1
850 * S1 S2 S2
851 * S1 S1 S1
852 * N N N
853 * N S1+S2 S2
854 * N S2 S2
855 * N S1 S1
856 *
857 * Note that you can't actually request stage-2 mappings.
858 */
859 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
860 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
861 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
862 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
863
Robin Murphy7602b872016-04-28 17:12:09 +0100864 /*
865 * Choosing a suitable context format is even more fiddly. Until we
866 * grow some way for the caller to express a preference, and/or move
867 * the decision into the io-pgtable code where it arguably belongs,
868 * just aim for the closest thing to the rest of the system, and hope
869 * that the hardware isn't esoteric enough that we can't assume AArch64
870 * support to be a superset of AArch32 support...
871 */
872 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
873 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
Robin Murphy60705292016-08-11 17:44:06 +0100874 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
875 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
876 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
877 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
878 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
Robin Murphy7602b872016-04-28 17:12:09 +0100879 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
880 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
881 ARM_SMMU_FEAT_FMT_AARCH64_16K |
882 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
883 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
884
885 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
886 ret = -EINVAL;
887 goto out_unlock;
888 }
889
Will Deaconc752ce42014-06-25 22:46:31 +0100890 switch (smmu_domain->stage) {
891 case ARM_SMMU_DOMAIN_S1:
892 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
893 start = smmu->num_s2_context_banks;
Will Deacon518f7132014-11-14 17:17:54 +0000894 ias = smmu->va_size;
895 oas = smmu->ipa_size;
Robin Murphy7602b872016-04-28 17:12:09 +0100896 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
Will Deacon518f7132014-11-14 17:17:54 +0000897 fmt = ARM_64_LPAE_S1;
Robin Murphy60705292016-08-11 17:44:06 +0100898 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
Will Deacon518f7132014-11-14 17:17:54 +0000899 fmt = ARM_32_LPAE_S1;
Robin Murphy7602b872016-04-28 17:12:09 +0100900 ias = min(ias, 32UL);
901 oas = min(oas, 40UL);
Robin Murphy60705292016-08-11 17:44:06 +0100902 } else {
903 fmt = ARM_V7S;
904 ias = min(ias, 32UL);
905 oas = min(oas, 32UL);
Robin Murphy7602b872016-04-28 17:12:09 +0100906 }
Will Deaconc752ce42014-06-25 22:46:31 +0100907 break;
908 case ARM_SMMU_DOMAIN_NESTED:
Will Deacon45ae7cf2013-06-24 18:31:25 +0100909 /*
910 * We will likely want to change this if/when KVM gets
911 * involved.
912 */
Will Deaconc752ce42014-06-25 22:46:31 +0100913 case ARM_SMMU_DOMAIN_S2:
Will Deacon9c5c92e2014-06-25 12:12:41 +0100914 cfg->cbar = CBAR_TYPE_S2_TRANS;
915 start = 0;
Will Deacon518f7132014-11-14 17:17:54 +0000916 ias = smmu->ipa_size;
917 oas = smmu->pa_size;
Robin Murphy7602b872016-04-28 17:12:09 +0100918 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
Will Deacon518f7132014-11-14 17:17:54 +0000919 fmt = ARM_64_LPAE_S2;
Robin Murphy7602b872016-04-28 17:12:09 +0100920 } else {
Will Deacon518f7132014-11-14 17:17:54 +0000921 fmt = ARM_32_LPAE_S2;
Robin Murphy7602b872016-04-28 17:12:09 +0100922 ias = min(ias, 40UL);
923 oas = min(oas, 40UL);
924 }
Will Deaconc752ce42014-06-25 22:46:31 +0100925 break;
926 default:
927 ret = -EINVAL;
928 goto out_unlock;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100929 }
930
931 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
932 smmu->num_context_banks);
Arnd Bergmann287980e2016-05-27 23:23:25 +0200933 if (ret < 0)
Mitchel Humpherysa18037b2014-07-30 18:58:13 +0100934 goto out_unlock;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100935
Will Deacon44680ee2014-06-25 11:29:12 +0100936 cfg->cbndx = ret;
Robin Murphyb7862e32016-04-13 18:13:03 +0100937 if (smmu->version < ARM_SMMU_V2) {
Will Deacon44680ee2014-06-25 11:29:12 +0100938 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
939 cfg->irptndx %= smmu->num_context_irqs;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100940 } else {
Will Deacon44680ee2014-06-25 11:29:12 +0100941 cfg->irptndx = cfg->cbndx;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100942 }
943
Will Deacon518f7132014-11-14 17:17:54 +0000944 pgtbl_cfg = (struct io_pgtable_cfg) {
Robin Murphyd5466352016-05-09 17:20:09 +0100945 .pgsize_bitmap = smmu->pgsize_bitmap,
Will Deacon518f7132014-11-14 17:17:54 +0000946 .ias = ias,
947 .oas = oas,
948 .tlb = &arm_smmu_gather_ops,
Robin Murphy2df7a252015-07-29 19:46:06 +0100949 .iommu_dev = smmu->dev,
Will Deacon518f7132014-11-14 17:17:54 +0000950 };
Mitchel Humpherysa18037b2014-07-30 18:58:13 +0100951
Will Deacon518f7132014-11-14 17:17:54 +0000952 smmu_domain->smmu = smmu;
953 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
954 if (!pgtbl_ops) {
955 ret = -ENOMEM;
956 goto out_clear_smmu;
957 }
958
Robin Murphyd5466352016-05-09 17:20:09 +0100959 /* Update the domain's page sizes to reflect the page table format */
960 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
Robin Murphy455eb7d2016-09-12 17:13:58 +0100961 domain->geometry.aperture_end = (1UL << ias) - 1;
962 domain->geometry.force_aperture = true;
Will Deacon518f7132014-11-14 17:17:54 +0000963
964 /* Initialise the context bank with our page table cfg */
965 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
966
967 /*
968 * Request context fault interrupt. Do this last to avoid the
969 * handler seeing a half-initialised domain state.
970 */
Will Deacon44680ee2014-06-25 11:29:12 +0100971 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
Peng Fanbee14002016-07-04 17:38:22 +0800972 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
973 IRQF_SHARED, "arm-smmu-context-fault", domain);
Arnd Bergmann287980e2016-05-27 23:23:25 +0200974 if (ret < 0) {
Will Deacon45ae7cf2013-06-24 18:31:25 +0100975 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
Will Deacon44680ee2014-06-25 11:29:12 +0100976 cfg->irptndx, irq);
977 cfg->irptndx = INVALID_IRPTNDX;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100978 }
979
Will Deacon518f7132014-11-14 17:17:54 +0000980 mutex_unlock(&smmu_domain->init_mutex);
981
982 /* Publish page table ops for map/unmap */
983 smmu_domain->pgtbl_ops = pgtbl_ops;
Will Deacona9a1b0b2014-05-01 18:05:08 +0100984 return 0;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100985
Will Deacon518f7132014-11-14 17:17:54 +0000986out_clear_smmu:
987 smmu_domain->smmu = NULL;
Mitchel Humpherysa18037b2014-07-30 18:58:13 +0100988out_unlock:
Will Deacon518f7132014-11-14 17:17:54 +0000989 mutex_unlock(&smmu_domain->init_mutex);
Will Deacon45ae7cf2013-06-24 18:31:25 +0100990 return ret;
991}
992
993static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
994{
Joerg Roedel1d672632015-03-26 13:43:10 +0100995 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon44680ee2014-06-25 11:29:12 +0100996 struct arm_smmu_device *smmu = smmu_domain->smmu;
997 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
Will Deacon1463fe42013-07-31 19:21:27 +0100998 void __iomem *cb_base;
Will Deacon45ae7cf2013-06-24 18:31:25 +0100999 int irq;
1000
Robin Murphy021bb842016-09-14 15:26:46 +01001001 if (!smmu)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001002 return;
1003
Will Deacon518f7132014-11-14 17:17:54 +00001004 /*
1005 * Disable the context bank and free the page tables before freeing
1006 * it.
1007 */
Will Deacon44680ee2014-06-25 11:29:12 +01001008 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
Will Deacon1463fe42013-07-31 19:21:27 +01001009 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
Will Deacon1463fe42013-07-31 19:21:27 +01001010
Will Deacon44680ee2014-06-25 11:29:12 +01001011 if (cfg->irptndx != INVALID_IRPTNDX) {
1012 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
Peng Fanbee14002016-07-04 17:38:22 +08001013 devm_free_irq(smmu->dev, irq, domain);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001014 }
1015
Markus Elfring44830b02015-11-06 18:32:41 +01001016 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
Will Deacon44680ee2014-06-25 11:29:12 +01001017 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001018}
1019
Joerg Roedel1d672632015-03-26 13:43:10 +01001020static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001021{
1022 struct arm_smmu_domain *smmu_domain;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001023
Robin Murphy9adb9592016-01-26 18:06:36 +00001024 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
Joerg Roedel1d672632015-03-26 13:43:10 +01001025 return NULL;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001026 /*
1027 * Allocate the domain and initialise some of its data structures.
1028 * We can't really do anything meaningful until we've added a
1029 * master.
1030 */
1031 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1032 if (!smmu_domain)
Joerg Roedel1d672632015-03-26 13:43:10 +01001033 return NULL;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001034
Robin Murphy021bb842016-09-14 15:26:46 +01001035 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1036 iommu_get_dma_cookie(&smmu_domain->domain))) {
Robin Murphy9adb9592016-01-26 18:06:36 +00001037 kfree(smmu_domain);
1038 return NULL;
1039 }
1040
Will Deacon518f7132014-11-14 17:17:54 +00001041 mutex_init(&smmu_domain->init_mutex);
1042 spin_lock_init(&smmu_domain->pgtbl_lock);
Joerg Roedel1d672632015-03-26 13:43:10 +01001043
1044 return &smmu_domain->domain;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001045}
1046
Joerg Roedel1d672632015-03-26 13:43:10 +01001047static void arm_smmu_domain_free(struct iommu_domain *domain)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001048{
Joerg Roedel1d672632015-03-26 13:43:10 +01001049 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon1463fe42013-07-31 19:21:27 +01001050
1051 /*
1052 * Free the domain resources. We assume that all devices have
1053 * already been detached.
1054 */
Robin Murphy9adb9592016-01-26 18:06:36 +00001055 iommu_put_dma_cookie(domain);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001056 arm_smmu_destroy_domain_context(domain);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001057 kfree(smmu_domain);
1058}
1059
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001060static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1061{
1062 struct arm_smmu_smr *smr = smmu->smrs + idx;
Robin Murphyf80cd882016-09-14 15:21:39 +01001063 u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001064
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001065 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001066 reg |= SMR_VALID;
1067 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1068}
1069
Robin Murphy8e8b2032016-09-12 17:13:50 +01001070static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1071{
1072 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1073 u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1074 (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1075 (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1076
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001077 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1078 smmu->smrs[idx].valid)
1079 reg |= S2CR_EXIDVALID;
Robin Murphy8e8b2032016-09-12 17:13:50 +01001080 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1081}
1082
1083static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1084{
1085 arm_smmu_write_s2cr(smmu, idx);
1086 if (smmu->smrs)
1087 arm_smmu_write_smr(smmu, idx);
1088}
1089
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001090/*
1091 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1092 * should be called after sCR0 is written.
1093 */
1094static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1095{
1096 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1097 u32 smr;
1098
1099 if (!smmu->smrs)
1100 return;
1101
1102 /*
1103 * SMR.ID bits may not be preserved if the corresponding MASK
1104 * bits are set, so check each one separately. We can reject
1105 * masters later if they try to claim IDs outside these masks.
1106 */
1107 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1108 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1109 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1110 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1111
1112 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1113 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1114 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1115 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1116}
1117
Robin Murphy588888a2016-09-12 17:13:54 +01001118static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001119{
1120 struct arm_smmu_smr *smrs = smmu->smrs;
Robin Murphy588888a2016-09-12 17:13:54 +01001121 int i, free_idx = -ENOSPC;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001122
Robin Murphy588888a2016-09-12 17:13:54 +01001123 /* Stream indexing is blissfully easy */
1124 if (!smrs)
1125 return id;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001126
Robin Murphy588888a2016-09-12 17:13:54 +01001127 /* Validating SMRs is... less so */
1128 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1129 if (!smrs[i].valid) {
1130 /*
1131 * Note the first free entry we come across, which
1132 * we'll claim in the end if nothing else matches.
1133 */
1134 if (free_idx < 0)
1135 free_idx = i;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001136 continue;
1137 }
Robin Murphy588888a2016-09-12 17:13:54 +01001138 /*
1139 * If the new entry is _entirely_ matched by an existing entry,
1140 * then reuse that, with the guarantee that there also cannot
1141 * be any subsequent conflicting entries. In normal use we'd
1142 * expect simply identical entries for this case, but there's
1143 * no harm in accommodating the generalisation.
1144 */
1145 if ((mask & smrs[i].mask) == mask &&
1146 !((id ^ smrs[i].id) & ~smrs[i].mask))
1147 return i;
1148 /*
1149 * If the new entry has any other overlap with an existing one,
1150 * though, then there always exists at least one stream ID
1151 * which would cause a conflict, and we can't allow that risk.
1152 */
1153 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1154 return -EINVAL;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001155 }
1156
Robin Murphy588888a2016-09-12 17:13:54 +01001157 return free_idx;
1158}
1159
1160static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1161{
1162 if (--smmu->s2crs[idx].count)
1163 return false;
1164
1165 smmu->s2crs[idx] = s2cr_init_val;
1166 if (smmu->smrs)
1167 smmu->smrs[idx].valid = false;
1168
1169 return true;
1170}
1171
1172static int arm_smmu_master_alloc_smes(struct device *dev)
1173{
Robin Murphyadfec2e2016-09-12 17:13:55 +01001174 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1175 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
Robin Murphy588888a2016-09-12 17:13:54 +01001176 struct arm_smmu_device *smmu = cfg->smmu;
1177 struct arm_smmu_smr *smrs = smmu->smrs;
1178 struct iommu_group *group;
1179 int i, idx, ret;
1180
1181 mutex_lock(&smmu->stream_map_mutex);
1182 /* Figure out a viable stream map entry allocation */
Robin Murphyadfec2e2016-09-12 17:13:55 +01001183 for_each_cfg_sme(fwspec, i, idx) {
Robin Murphy021bb842016-09-14 15:26:46 +01001184 u16 sid = fwspec->ids[i];
1185 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1186
Robin Murphy588888a2016-09-12 17:13:54 +01001187 if (idx != INVALID_SMENDX) {
1188 ret = -EEXIST;
1189 goto out_err;
1190 }
1191
Robin Murphy021bb842016-09-14 15:26:46 +01001192 ret = arm_smmu_find_sme(smmu, sid, mask);
Robin Murphy588888a2016-09-12 17:13:54 +01001193 if (ret < 0)
1194 goto out_err;
1195
1196 idx = ret;
1197 if (smrs && smmu->s2crs[idx].count == 0) {
Robin Murphy021bb842016-09-14 15:26:46 +01001198 smrs[idx].id = sid;
1199 smrs[idx].mask = mask;
Robin Murphy588888a2016-09-12 17:13:54 +01001200 smrs[idx].valid = true;
1201 }
1202 smmu->s2crs[idx].count++;
1203 cfg->smendx[i] = (s16)idx;
1204 }
1205
1206 group = iommu_group_get_for_dev(dev);
1207 if (!group)
1208 group = ERR_PTR(-ENOMEM);
1209 if (IS_ERR(group)) {
1210 ret = PTR_ERR(group);
1211 goto out_err;
1212 }
1213 iommu_group_put(group);
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001214
Will Deacon45ae7cf2013-06-24 18:31:25 +01001215 /* It worked! Now, poke the actual hardware */
Robin Murphyadfec2e2016-09-12 17:13:55 +01001216 for_each_cfg_sme(fwspec, i, idx) {
Robin Murphy588888a2016-09-12 17:13:54 +01001217 arm_smmu_write_sme(smmu, idx);
1218 smmu->s2crs[idx].group = group;
1219 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01001220
Robin Murphy588888a2016-09-12 17:13:54 +01001221 mutex_unlock(&smmu->stream_map_mutex);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001222 return 0;
1223
Robin Murphy588888a2016-09-12 17:13:54 +01001224out_err:
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001225 while (i--) {
Robin Murphy588888a2016-09-12 17:13:54 +01001226 arm_smmu_free_sme(smmu, cfg->smendx[i]);
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001227 cfg->smendx[i] = INVALID_SMENDX;
1228 }
Robin Murphy588888a2016-09-12 17:13:54 +01001229 mutex_unlock(&smmu->stream_map_mutex);
1230 return ret;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001231}
1232
Robin Murphyadfec2e2016-09-12 17:13:55 +01001233static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001234{
Robin Murphyadfec2e2016-09-12 17:13:55 +01001235 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1236 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
Robin Murphyd3097e32016-09-12 17:13:53 +01001237 int i, idx;
Will Deacon43b412b2014-07-15 11:22:24 +01001238
Robin Murphy588888a2016-09-12 17:13:54 +01001239 mutex_lock(&smmu->stream_map_mutex);
Robin Murphyadfec2e2016-09-12 17:13:55 +01001240 for_each_cfg_sme(fwspec, i, idx) {
Robin Murphy588888a2016-09-12 17:13:54 +01001241 if (arm_smmu_free_sme(smmu, idx))
1242 arm_smmu_write_sme(smmu, idx);
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001243 cfg->smendx[i] = INVALID_SMENDX;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001244 }
Robin Murphy588888a2016-09-12 17:13:54 +01001245 mutex_unlock(&smmu->stream_map_mutex);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001246}
1247
Will Deacon45ae7cf2013-06-24 18:31:25 +01001248static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
Robin Murphyadfec2e2016-09-12 17:13:55 +01001249 struct iommu_fwspec *fwspec)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001250{
Will Deacon44680ee2014-06-25 11:29:12 +01001251 struct arm_smmu_device *smmu = smmu_domain->smmu;
Robin Murphy8e8b2032016-09-12 17:13:50 +01001252 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1253 enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1254 u8 cbndx = smmu_domain->cfg.cbndx;
Robin Murphy588888a2016-09-12 17:13:54 +01001255 int i, idx;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001256
Robin Murphyadfec2e2016-09-12 17:13:55 +01001257 for_each_cfg_sme(fwspec, i, idx) {
Robin Murphy8e8b2032016-09-12 17:13:50 +01001258 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
Robin Murphy588888a2016-09-12 17:13:54 +01001259 continue;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001260
Robin Murphy8e8b2032016-09-12 17:13:50 +01001261 s2cr[idx].type = type;
Sricharan Re1989802017-01-06 18:58:15 +05301262 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
Robin Murphy8e8b2032016-09-12 17:13:50 +01001263 s2cr[idx].cbndx = cbndx;
1264 arm_smmu_write_s2cr(smmu, idx);
Will Deacon43b412b2014-07-15 11:22:24 +01001265 }
Robin Murphy8e8b2032016-09-12 17:13:50 +01001266 return 0;
Will Deaconbc7f2ce2016-02-17 17:41:57 +00001267}
1268
Will Deacon45ae7cf2013-06-24 18:31:25 +01001269static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1270{
Mitchel Humpherysa18037b2014-07-30 18:58:13 +01001271 int ret;
Robin Murphyadfec2e2016-09-12 17:13:55 +01001272 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1273 struct arm_smmu_device *smmu;
Joerg Roedel1d672632015-03-26 13:43:10 +01001274 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001275
Robin Murphyadfec2e2016-09-12 17:13:55 +01001276 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001277 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1278 return -ENXIO;
1279 }
1280
Robin Murphyfba4f8e2016-10-17 12:06:21 +01001281 /*
1282 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1283 * domains between of_xlate() and add_device() - we have no way to cope
1284 * with that, so until ARM gets converted to rely on groups and default
1285 * domains, just say no (but more politely than by dereferencing NULL).
1286 * This should be at least a WARN_ON once that's sorted.
1287 */
1288 if (!fwspec->iommu_priv)
1289 return -ENODEV;
1290
Robin Murphyadfec2e2016-09-12 17:13:55 +01001291 smmu = fwspec_smmu(fwspec);
Will Deacon518f7132014-11-14 17:17:54 +00001292 /* Ensure that the domain is finalised */
Robin Murphyadfec2e2016-09-12 17:13:55 +01001293 ret = arm_smmu_init_domain_context(domain, smmu);
Arnd Bergmann287980e2016-05-27 23:23:25 +02001294 if (ret < 0)
Will Deacon518f7132014-11-14 17:17:54 +00001295 return ret;
1296
Will Deacon45ae7cf2013-06-24 18:31:25 +01001297 /*
Will Deacon44680ee2014-06-25 11:29:12 +01001298 * Sanity check the domain. We don't support domains across
1299 * different SMMUs.
Will Deacon45ae7cf2013-06-24 18:31:25 +01001300 */
Robin Murphyadfec2e2016-09-12 17:13:55 +01001301 if (smmu_domain->smmu != smmu) {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001302 dev_err(dev,
1303 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
Robin Murphyadfec2e2016-09-12 17:13:55 +01001304 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
Mitchel Humpherysa18037b2014-07-30 18:58:13 +01001305 return -EINVAL;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001306 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01001307
1308 /* Looks ok, so add the device to the domain */
Robin Murphyadfec2e2016-09-12 17:13:55 +01001309 return arm_smmu_domain_add_master(smmu_domain, fwspec);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001310}
1311
Will Deacon45ae7cf2013-06-24 18:31:25 +01001312static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
Will Deaconb410aed2014-02-20 16:31:06 +00001313 phys_addr_t paddr, size_t size, int prot)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001314{
Will Deacon518f7132014-11-14 17:17:54 +00001315 int ret;
1316 unsigned long flags;
Joerg Roedel1d672632015-03-26 13:43:10 +01001317 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon518f7132014-11-14 17:17:54 +00001318 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001319
Will Deacon518f7132014-11-14 17:17:54 +00001320 if (!ops)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001321 return -ENODEV;
1322
Will Deacon518f7132014-11-14 17:17:54 +00001323 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1324 ret = ops->map(ops, iova, paddr, size, prot);
1325 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1326 return ret;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001327}
1328
1329static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1330 size_t size)
1331{
Will Deacon518f7132014-11-14 17:17:54 +00001332 size_t ret;
1333 unsigned long flags;
Joerg Roedel1d672632015-03-26 13:43:10 +01001334 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon518f7132014-11-14 17:17:54 +00001335 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001336
Will Deacon518f7132014-11-14 17:17:54 +00001337 if (!ops)
1338 return 0;
1339
1340 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1341 ret = ops->unmap(ops, iova, size);
1342 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1343 return ret;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001344}
1345
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001346static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1347 dma_addr_t iova)
1348{
Joerg Roedel1d672632015-03-26 13:43:10 +01001349 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001350 struct arm_smmu_device *smmu = smmu_domain->smmu;
1351 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1352 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1353 struct device *dev = smmu->dev;
1354 void __iomem *cb_base;
1355 u32 tmp;
1356 u64 phys;
Robin Murphy661d9622015-05-27 17:09:34 +01001357 unsigned long va;
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001358
1359 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1360
Robin Murphy661d9622015-05-27 17:09:34 +01001361 /* ATS1 registers can only be written atomically */
1362 va = iova & ~0xfffUL;
Robin Murphy661d9622015-05-27 17:09:34 +01001363 if (smmu->version == ARM_SMMU_V2)
Robin Murphyf9a05f02016-04-13 18:13:01 +01001364 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1365 else /* Register is only 32-bit in v1 */
Robin Murphy661d9622015-05-27 17:09:34 +01001366 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001367
1368 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1369 !(tmp & ATSR_ACTIVE), 5, 50)) {
1370 dev_err(dev,
Fabio Estevam077124c2015-08-18 17:12:24 +01001371 "iova to phys timed out on %pad. Falling back to software table walk.\n",
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001372 &iova);
1373 return ops->iova_to_phys(ops, iova);
1374 }
1375
Robin Murphyf9a05f02016-04-13 18:13:01 +01001376 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001377 if (phys & CB_PAR_F) {
1378 dev_err(dev, "translation fault!\n");
1379 dev_err(dev, "PAR = 0x%llx\n", phys);
1380 return 0;
1381 }
1382
1383 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1384}
1385
Will Deacon45ae7cf2013-06-24 18:31:25 +01001386static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001387 dma_addr_t iova)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001388{
Will Deacon518f7132014-11-14 17:17:54 +00001389 phys_addr_t ret;
1390 unsigned long flags;
Joerg Roedel1d672632015-03-26 13:43:10 +01001391 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deacon518f7132014-11-14 17:17:54 +00001392 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001393
Will Deacon518f7132014-11-14 17:17:54 +00001394 if (!ops)
Will Deacona44a9792013-11-07 18:47:50 +00001395 return 0;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001396
Will Deacon518f7132014-11-14 17:17:54 +00001397 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
Baptiste Reynal83a60ed2015-03-04 16:51:06 +01001398 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1399 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001400 ret = arm_smmu_iova_to_phys_hard(domain, iova);
Baptiste Reynal83a60ed2015-03-04 16:51:06 +01001401 } else {
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001402 ret = ops->iova_to_phys(ops, iova);
Baptiste Reynal83a60ed2015-03-04 16:51:06 +01001403 }
1404
Will Deacon518f7132014-11-14 17:17:54 +00001405 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001406
Will Deacon518f7132014-11-14 17:17:54 +00001407 return ret;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001408}
1409
Joerg Roedel1fd0c772014-09-05 10:49:34 +02001410static bool arm_smmu_capable(enum iommu_cap cap)
Will Deacon45ae7cf2013-06-24 18:31:25 +01001411{
Will Deacond0948942014-06-24 17:30:10 +01001412 switch (cap) {
1413 case IOMMU_CAP_CACHE_COHERENCY:
Joerg Roedel1fd0c772014-09-05 10:49:34 +02001414 /*
1415 * Return true here as the SMMU can always send out coherent
1416 * requests.
1417 */
1418 return true;
Antonios Motakis0029a8d2014-10-13 14:06:18 +01001419 case IOMMU_CAP_NOEXEC:
1420 return true;
Will Deacond0948942014-06-24 17:30:10 +01001421 default:
Joerg Roedel1fd0c772014-09-05 10:49:34 +02001422 return false;
Will Deacond0948942014-06-24 17:30:10 +01001423 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01001424}
Will Deacon45ae7cf2013-06-24 18:31:25 +01001425
Robin Murphy021bb842016-09-14 15:26:46 +01001426static int arm_smmu_match_node(struct device *dev, void *data)
1427{
Lorenzo Pieralisice9babe2016-11-21 10:01:37 +00001428 return dev->fwnode == data;
Robin Murphy021bb842016-09-14 15:26:46 +01001429}
1430
Lorenzo Pieralisice9babe2016-11-21 10:01:37 +00001431static
1432struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
Robin Murphy021bb842016-09-14 15:26:46 +01001433{
1434 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
Lorenzo Pieralisice9babe2016-11-21 10:01:37 +00001435 fwnode, arm_smmu_match_node);
Robin Murphy021bb842016-09-14 15:26:46 +01001436 put_device(dev);
1437 return dev ? dev_get_drvdata(dev) : NULL;
1438}
1439
Will Deacon03edb222015-01-19 14:27:33 +00001440static int arm_smmu_add_device(struct device *dev)
1441{
Robin Murphyadfec2e2016-09-12 17:13:55 +01001442 struct arm_smmu_device *smmu;
Robin Murphyf80cd882016-09-14 15:21:39 +01001443 struct arm_smmu_master_cfg *cfg;
Robin Murphy021bb842016-09-14 15:26:46 +01001444 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
Robin Murphyf80cd882016-09-14 15:21:39 +01001445 int i, ret;
1446
Robin Murphy021bb842016-09-14 15:26:46 +01001447 if (using_legacy_binding) {
1448 ret = arm_smmu_register_legacy_master(dev, &smmu);
1449 fwspec = dev->iommu_fwspec;
1450 if (ret)
1451 goto out_free;
Robin Murphy3c117b52016-11-02 17:31:32 +00001452 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
Lorenzo Pieralisice9babe2016-11-21 10:01:37 +00001453 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
Robin Murphy021bb842016-09-14 15:26:46 +01001454 } else {
1455 return -ENODEV;
1456 }
Robin Murphyf80cd882016-09-14 15:21:39 +01001457
1458 ret = -EINVAL;
Robin Murphyadfec2e2016-09-12 17:13:55 +01001459 for (i = 0; i < fwspec->num_ids; i++) {
1460 u16 sid = fwspec->ids[i];
Robin Murphy021bb842016-09-14 15:26:46 +01001461 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
Robin Murphyf80cd882016-09-14 15:21:39 +01001462
Robin Murphyadfec2e2016-09-12 17:13:55 +01001463 if (sid & ~smmu->streamid_mask) {
Robin Murphyf80cd882016-09-14 15:21:39 +01001464 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
Robin Murphy021bb842016-09-14 15:26:46 +01001465 sid, smmu->streamid_mask);
1466 goto out_free;
1467 }
1468 if (mask & ~smmu->smr_mask_mask) {
1469 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1470 sid, smmu->smr_mask_mask);
Robin Murphyf80cd882016-09-14 15:21:39 +01001471 goto out_free;
1472 }
Robin Murphyf80cd882016-09-14 15:21:39 +01001473 }
Will Deacon03edb222015-01-19 14:27:33 +00001474
Robin Murphyadfec2e2016-09-12 17:13:55 +01001475 ret = -ENOMEM;
1476 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1477 GFP_KERNEL);
1478 if (!cfg)
1479 goto out_free;
1480
1481 cfg->smmu = smmu;
1482 fwspec->iommu_priv = cfg;
1483 while (i--)
1484 cfg->smendx[i] = INVALID_SMENDX;
1485
Robin Murphy588888a2016-09-12 17:13:54 +01001486 ret = arm_smmu_master_alloc_smes(dev);
Robin Murphyadfec2e2016-09-12 17:13:55 +01001487 if (ret)
1488 goto out_free;
1489
Joerg Roedel9648cbc2017-02-01 18:11:36 +01001490 iommu_device_link(&smmu->iommu, dev);
1491
Robin Murphyadfec2e2016-09-12 17:13:55 +01001492 return 0;
Robin Murphyf80cd882016-09-14 15:21:39 +01001493
1494out_free:
Robin Murphyadfec2e2016-09-12 17:13:55 +01001495 if (fwspec)
1496 kfree(fwspec->iommu_priv);
1497 iommu_fwspec_free(dev);
Robin Murphyf80cd882016-09-14 15:21:39 +01001498 return ret;
Will Deacon03edb222015-01-19 14:27:33 +00001499}
1500
Will Deacon45ae7cf2013-06-24 18:31:25 +01001501static void arm_smmu_remove_device(struct device *dev)
1502{
Robin Murphyadfec2e2016-09-12 17:13:55 +01001503 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
Joerg Roedel9648cbc2017-02-01 18:11:36 +01001504 struct arm_smmu_master_cfg *cfg;
1505 struct arm_smmu_device *smmu;
1506
Robin Murphy8e8b2032016-09-12 17:13:50 +01001507
Robin Murphyadfec2e2016-09-12 17:13:55 +01001508 if (!fwspec || fwspec->ops != &arm_smmu_ops)
Robin Murphyf80cd882016-09-14 15:21:39 +01001509 return;
Robin Murphy8e8b2032016-09-12 17:13:50 +01001510
Joerg Roedel9648cbc2017-02-01 18:11:36 +01001511 cfg = fwspec->iommu_priv;
1512 smmu = cfg->smmu;
1513
1514 iommu_device_unlink(&smmu->iommu, dev);
Robin Murphyadfec2e2016-09-12 17:13:55 +01001515 arm_smmu_master_free_smes(fwspec);
Antonios Motakis5fc63a72013-10-18 16:08:29 +01001516 iommu_group_remove_device(dev);
Robin Murphyadfec2e2016-09-12 17:13:55 +01001517 kfree(fwspec->iommu_priv);
1518 iommu_fwspec_free(dev);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001519}
1520
Joerg Roedelaf659932015-10-21 23:51:41 +02001521static struct iommu_group *arm_smmu_device_group(struct device *dev)
1522{
Robin Murphyadfec2e2016-09-12 17:13:55 +01001523 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1524 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
Robin Murphy588888a2016-09-12 17:13:54 +01001525 struct iommu_group *group = NULL;
1526 int i, idx;
1527
Robin Murphyadfec2e2016-09-12 17:13:55 +01001528 for_each_cfg_sme(fwspec, i, idx) {
Robin Murphy588888a2016-09-12 17:13:54 +01001529 if (group && smmu->s2crs[idx].group &&
1530 group != smmu->s2crs[idx].group)
1531 return ERR_PTR(-EINVAL);
1532
1533 group = smmu->s2crs[idx].group;
1534 }
1535
1536 if (group)
Robin Murphye1b44cb2016-11-11 17:59:22 +00001537 return iommu_group_ref_get(group);
Joerg Roedelaf659932015-10-21 23:51:41 +02001538
1539 if (dev_is_pci(dev))
1540 group = pci_device_group(dev);
1541 else
1542 group = generic_device_group(dev);
1543
Joerg Roedelaf659932015-10-21 23:51:41 +02001544 return group;
1545}
1546
Will Deaconc752ce42014-06-25 22:46:31 +01001547static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1548 enum iommu_attr attr, void *data)
1549{
Joerg Roedel1d672632015-03-26 13:43:10 +01001550 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deaconc752ce42014-06-25 22:46:31 +01001551
1552 switch (attr) {
1553 case DOMAIN_ATTR_NESTING:
1554 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1555 return 0;
1556 default:
1557 return -ENODEV;
1558 }
1559}
1560
1561static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1562 enum iommu_attr attr, void *data)
1563{
Will Deacon518f7132014-11-14 17:17:54 +00001564 int ret = 0;
Joerg Roedel1d672632015-03-26 13:43:10 +01001565 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
Will Deaconc752ce42014-06-25 22:46:31 +01001566
Will Deacon518f7132014-11-14 17:17:54 +00001567 mutex_lock(&smmu_domain->init_mutex);
1568
Will Deaconc752ce42014-06-25 22:46:31 +01001569 switch (attr) {
1570 case DOMAIN_ATTR_NESTING:
Will Deacon518f7132014-11-14 17:17:54 +00001571 if (smmu_domain->smmu) {
1572 ret = -EPERM;
1573 goto out_unlock;
1574 }
1575
Will Deaconc752ce42014-06-25 22:46:31 +01001576 if (*(int *)data)
1577 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1578 else
1579 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1580
Will Deacon518f7132014-11-14 17:17:54 +00001581 break;
Will Deaconc752ce42014-06-25 22:46:31 +01001582 default:
Will Deacon518f7132014-11-14 17:17:54 +00001583 ret = -ENODEV;
Will Deaconc752ce42014-06-25 22:46:31 +01001584 }
Will Deacon518f7132014-11-14 17:17:54 +00001585
1586out_unlock:
1587 mutex_unlock(&smmu_domain->init_mutex);
1588 return ret;
Will Deaconc752ce42014-06-25 22:46:31 +01001589}
1590
Robin Murphy021bb842016-09-14 15:26:46 +01001591static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1592{
1593 u32 fwid = 0;
1594
1595 if (args->args_count > 0)
1596 fwid |= (u16)args->args[0];
1597
1598 if (args->args_count > 1)
1599 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1600
1601 return iommu_fwspec_add_ids(dev, &fwid, 1);
1602}
1603
Eric Augerf3ebee82017-01-19 20:57:55 +00001604static void arm_smmu_get_resv_regions(struct device *dev,
1605 struct list_head *head)
1606{
1607 struct iommu_resv_region *region;
1608 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1609
1610 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
Robin Murphy9d3a4de2017-03-16 17:00:16 +00001611 prot, IOMMU_RESV_SW_MSI);
Eric Augerf3ebee82017-01-19 20:57:55 +00001612 if (!region)
1613 return;
1614
1615 list_add_tail(&region->list, head);
1616}
1617
1618static void arm_smmu_put_resv_regions(struct device *dev,
1619 struct list_head *head)
1620{
1621 struct iommu_resv_region *entry, *next;
1622
1623 list_for_each_entry_safe(entry, next, head, list)
1624 kfree(entry);
1625}
1626
Will Deacon518f7132014-11-14 17:17:54 +00001627static struct iommu_ops arm_smmu_ops = {
Will Deaconc752ce42014-06-25 22:46:31 +01001628 .capable = arm_smmu_capable,
Joerg Roedel1d672632015-03-26 13:43:10 +01001629 .domain_alloc = arm_smmu_domain_alloc,
1630 .domain_free = arm_smmu_domain_free,
Will Deaconc752ce42014-06-25 22:46:31 +01001631 .attach_dev = arm_smmu_attach_dev,
Will Deaconc752ce42014-06-25 22:46:31 +01001632 .map = arm_smmu_map,
1633 .unmap = arm_smmu_unmap,
Joerg Roedel76771c92014-12-02 13:07:13 +01001634 .map_sg = default_iommu_map_sg,
Will Deaconc752ce42014-06-25 22:46:31 +01001635 .iova_to_phys = arm_smmu_iova_to_phys,
1636 .add_device = arm_smmu_add_device,
1637 .remove_device = arm_smmu_remove_device,
Joerg Roedelaf659932015-10-21 23:51:41 +02001638 .device_group = arm_smmu_device_group,
Will Deaconc752ce42014-06-25 22:46:31 +01001639 .domain_get_attr = arm_smmu_domain_get_attr,
1640 .domain_set_attr = arm_smmu_domain_set_attr,
Robin Murphy021bb842016-09-14 15:26:46 +01001641 .of_xlate = arm_smmu_of_xlate,
Eric Augerf3ebee82017-01-19 20:57:55 +00001642 .get_resv_regions = arm_smmu_get_resv_regions,
1643 .put_resv_regions = arm_smmu_put_resv_regions,
Will Deacon518f7132014-11-14 17:17:54 +00001644 .pgsize_bitmap = -1UL, /* Restricted during device attach */
Will Deacon45ae7cf2013-06-24 18:31:25 +01001645};
1646
1647static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1648{
1649 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001650 void __iomem *cb_base;
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001651 int i;
Peng Fan3ca37122016-05-03 21:50:30 +08001652 u32 reg, major;
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001653
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +00001654 /* clear global FSR */
1655 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1656 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001657
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001658 /*
1659 * Reset stream mapping groups: Initial values mark all SMRn as
1660 * invalid and all S2CRn as bypass unless overridden.
1661 */
Robin Murphy8e8b2032016-09-12 17:13:50 +01001662 for (i = 0; i < smmu->num_mapping_groups; ++i)
1663 arm_smmu_write_sme(smmu, i);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001664
Nipun Gupta6eb18d42016-11-04 15:25:23 +05301665 if (smmu->model == ARM_MMU500) {
1666 /*
1667 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1668 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1669 * bit is only present in MMU-500r2 onwards.
1670 */
1671 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1672 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
Peng Fan3ca37122016-05-03 21:50:30 +08001673 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
Nipun Gupta6eb18d42016-11-04 15:25:23 +05301674 if (major >= 2)
1675 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1676 /*
1677 * Allow unmatched Stream IDs to allocate bypass
1678 * TLB entries for reduced latency.
1679 */
1680 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
Peng Fan3ca37122016-05-03 21:50:30 +08001681 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1682 }
1683
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001684 /* Make sure all context banks are disabled and clear CB_FSR */
1685 for (i = 0; i < smmu->num_context_banks; ++i) {
1686 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1687 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1688 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
Robin Murphyf0cfffc2016-04-13 18:12:59 +01001689 /*
1690 * Disable MMU-500's not-particularly-beneficial next-page
1691 * prefetcher for the sake of errata #841119 and #826419.
1692 */
1693 if (smmu->model == ARM_MMU500) {
1694 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1695 reg &= ~ARM_MMU500_ACTLR_CPRE;
1696 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1697 }
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001698 }
Will Deacon1463fe42013-07-31 19:21:27 +01001699
Will Deacon45ae7cf2013-06-24 18:31:25 +01001700 /* Invalidate the TLB, just in case */
Will Deacon45ae7cf2013-06-24 18:31:25 +01001701 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1702 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1703
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +00001704 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001705
Will Deacon45ae7cf2013-06-24 18:31:25 +01001706 /* Enable fault reporting */
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001707 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001708
1709 /* Disable TLB broadcasting. */
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001710 reg |= (sCR0_VMIDPNE | sCR0_PTM);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001711
Robin Murphy25a1c962016-02-10 14:25:33 +00001712 /* Enable client access, handling unmatched streams as appropriate */
1713 reg &= ~sCR0_CLIENTPD;
1714 if (disable_bypass)
1715 reg |= sCR0_USFCFG;
1716 else
1717 reg &= ~sCR0_USFCFG;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001718
1719 /* Disable forced broadcasting */
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001720 reg &= ~sCR0_FB;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001721
1722 /* Don't upgrade barriers */
Andreas Herrmann659db6f2013-10-01 13:39:09 +01001723 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001724
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -08001725 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1726 reg |= sCR0_VMID16EN;
1727
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001728 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1729 reg |= sCR0_EXIDENABLE;
1730
Will Deacon45ae7cf2013-06-24 18:31:25 +01001731 /* Push the button */
Will Deacon518f7132014-11-14 17:17:54 +00001732 __arm_smmu_tlb_sync(smmu);
Andreas Herrmann3a5df8f2014-01-30 18:18:04 +00001733 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001734}
1735
1736static int arm_smmu_id_size_to_bits(int size)
1737{
1738 switch (size) {
1739 case 0:
1740 return 32;
1741 case 1:
1742 return 36;
1743 case 2:
1744 return 40;
1745 case 3:
1746 return 42;
1747 case 4:
1748 return 44;
1749 case 5:
1750 default:
1751 return 48;
1752 }
1753}
1754
1755static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1756{
1757 unsigned long size;
1758 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1759 u32 id;
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00001760 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
Robin Murphy8e8b2032016-09-12 17:13:50 +01001761 int i;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001762
1763 dev_notice(smmu->dev, "probing hardware configuration...\n");
Robin Murphyb7862e32016-04-13 18:13:03 +01001764 dev_notice(smmu->dev, "SMMUv%d with:\n",
1765 smmu->version == ARM_SMMU_V2 ? 2 : 1);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001766
1767 /* ID0 */
1768 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
Will Deacon4cf740b2014-07-14 19:47:39 +01001769
1770 /* Restrict available stages based on module parameter */
1771 if (force_stage == 1)
1772 id &= ~(ID0_S2TS | ID0_NTS);
1773 else if (force_stage == 2)
1774 id &= ~(ID0_S1TS | ID0_NTS);
1775
Will Deacon45ae7cf2013-06-24 18:31:25 +01001776 if (id & ID0_S1TS) {
1777 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1778 dev_notice(smmu->dev, "\tstage 1 translation\n");
1779 }
1780
1781 if (id & ID0_S2TS) {
1782 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1783 dev_notice(smmu->dev, "\tstage 2 translation\n");
1784 }
1785
1786 if (id & ID0_NTS) {
1787 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1788 dev_notice(smmu->dev, "\tnested translation\n");
1789 }
1790
1791 if (!(smmu->features &
Will Deacon4cf740b2014-07-14 19:47:39 +01001792 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001793 dev_err(smmu->dev, "\tno translation support!\n");
1794 return -ENODEV;
1795 }
1796
Robin Murphyb7862e32016-04-13 18:13:03 +01001797 if ((id & ID0_S1TS) &&
1798 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
Mitchel Humpherys859a7322014-10-29 21:13:40 +00001799 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1800 dev_notice(smmu->dev, "\taddress translation ops\n");
1801 }
1802
Robin Murphybae2c2d2015-07-29 19:46:05 +01001803 /*
1804 * In order for DMA API calls to work properly, we must defer to what
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00001805 * the FW says about coherency, regardless of what the hardware claims.
Robin Murphybae2c2d2015-07-29 19:46:05 +01001806 * Fortunately, this also opens up a workaround for systems where the
1807 * ID register value has ended up configured incorrectly.
1808 */
Robin Murphybae2c2d2015-07-29 19:46:05 +01001809 cttw_reg = !!(id & ID0_CTTW);
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00001810 if (cttw_fw || cttw_reg)
Robin Murphybae2c2d2015-07-29 19:46:05 +01001811 dev_notice(smmu->dev, "\t%scoherent table walk\n",
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00001812 cttw_fw ? "" : "non-");
1813 if (cttw_fw != cttw_reg)
Robin Murphybae2c2d2015-07-29 19:46:05 +01001814 dev_notice(smmu->dev,
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00001815 "\t(IDR0.CTTW overridden by FW configuration)\n");
Will Deacon45ae7cf2013-06-24 18:31:25 +01001816
Robin Murphy21174242016-09-12 17:13:48 +01001817 /* Max. number of entries we have for stream matching/indexing */
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001818 if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1819 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1820 size = 1 << 16;
1821 } else {
1822 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1823 }
Robin Murphy21174242016-09-12 17:13:48 +01001824 smmu->streamid_mask = size - 1;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001825 if (id & ID0_SMS) {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001826 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
Robin Murphy21174242016-09-12 17:13:48 +01001827 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1828 if (size == 0) {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001829 dev_err(smmu->dev,
1830 "stream-matching supported, but no SMRs present!\n");
1831 return -ENODEV;
1832 }
1833
Robin Murphy1f3d5ca2016-09-12 17:13:49 +01001834 /* Zero-initialised to mark as invalid */
1835 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1836 GFP_KERNEL);
1837 if (!smmu->smrs)
1838 return -ENOMEM;
1839
Will Deacon45ae7cf2013-06-24 18:31:25 +01001840 dev_notice(smmu->dev,
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03001841 "\tstream matching with %lu register groups", size);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001842 }
Robin Murphy8e8b2032016-09-12 17:13:50 +01001843 /* s2cr->type == 0 means translation, so initialise explicitly */
1844 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1845 GFP_KERNEL);
1846 if (!smmu->s2crs)
1847 return -ENOMEM;
1848 for (i = 0; i < size; i++)
1849 smmu->s2crs[i] = s2cr_init_val;
1850
Robin Murphy21174242016-09-12 17:13:48 +01001851 smmu->num_mapping_groups = size;
Robin Murphy588888a2016-09-12 17:13:54 +01001852 mutex_init(&smmu->stream_map_mutex);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001853
Robin Murphy7602b872016-04-28 17:12:09 +01001854 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1855 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1856 if (!(id & ID0_PTFS_NO_AARCH32S))
1857 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1858 }
1859
Will Deacon45ae7cf2013-06-24 18:31:25 +01001860 /* ID1 */
1861 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
Will Deaconc757e852014-07-30 11:33:25 +01001862 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001863
Andreas Herrmannc55af7f2013-10-01 13:39:06 +01001864 /* Check for size mismatch of SMMU address space from mapped region */
Will Deacon518f7132014-11-14 17:17:54 +00001865 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
Will Deaconc757e852014-07-30 11:33:25 +01001866 size *= 2 << smmu->pgshift;
Andreas Herrmannc55af7f2013-10-01 13:39:06 +01001867 if (smmu->size != size)
Mitchel Humpherys29073202014-07-08 09:52:18 -07001868 dev_warn(smmu->dev,
1869 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1870 size, smmu->size);
Will Deacon45ae7cf2013-06-24 18:31:25 +01001871
Will Deacon518f7132014-11-14 17:17:54 +00001872 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001873 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1874 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1875 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1876 return -ENODEV;
1877 }
1878 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1879 smmu->num_context_banks, smmu->num_s2_context_banks);
Robin Murphye086d912016-04-13 18:12:58 +01001880 /*
1881 * Cavium CN88xx erratum #27704.
1882 * Ensure ASID and VMID allocation is unique across all SMMUs in
1883 * the system.
1884 */
1885 if (smmu->model == CAVIUM_SMMUV2) {
1886 smmu->cavium_id_base =
1887 atomic_add_return(smmu->num_context_banks,
1888 &cavium_smmu_context_count);
1889 smmu->cavium_id_base -= smmu->num_context_banks;
1890 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01001891
1892 /* ID2 */
1893 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1894 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
Will Deacon518f7132014-11-14 17:17:54 +00001895 smmu->ipa_size = size;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001896
Will Deacon518f7132014-11-14 17:17:54 +00001897 /* The output mask is also applied for bypass */
Will Deacon45ae7cf2013-06-24 18:31:25 +01001898 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
Will Deacon518f7132014-11-14 17:17:54 +00001899 smmu->pa_size = size;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001900
Tirumalesh Chalamarla4e3e9b62016-02-23 10:19:00 -08001901 if (id & ID2_VMID16)
1902 smmu->features |= ARM_SMMU_FEAT_VMID16;
1903
Robin Murphyf1d84542015-03-04 16:41:05 +00001904 /*
1905 * What the page table walker can address actually depends on which
1906 * descriptor format is in use, but since a) we don't know that yet,
1907 * and b) it can vary per context bank, this will have to do...
1908 */
1909 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1910 dev_warn(smmu->dev,
1911 "failed to set DMA mask for table walker\n");
1912
Robin Murphyb7862e32016-04-13 18:13:03 +01001913 if (smmu->version < ARM_SMMU_V2) {
Will Deacon518f7132014-11-14 17:17:54 +00001914 smmu->va_size = smmu->ipa_size;
Robin Murphyb7862e32016-04-13 18:13:03 +01001915 if (smmu->version == ARM_SMMU_V1_64K)
1916 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001917 } else {
Will Deacon45ae7cf2013-06-24 18:31:25 +01001918 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
Will Deacon518f7132014-11-14 17:17:54 +00001919 smmu->va_size = arm_smmu_id_size_to_bits(size);
Will Deacon518f7132014-11-14 17:17:54 +00001920 if (id & ID2_PTFS_4K)
Robin Murphy7602b872016-04-28 17:12:09 +01001921 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
Will Deacon518f7132014-11-14 17:17:54 +00001922 if (id & ID2_PTFS_16K)
Robin Murphy7602b872016-04-28 17:12:09 +01001923 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
Will Deacon518f7132014-11-14 17:17:54 +00001924 if (id & ID2_PTFS_64K)
Robin Murphy7602b872016-04-28 17:12:09 +01001925 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
Will Deacon45ae7cf2013-06-24 18:31:25 +01001926 }
1927
Robin Murphy7602b872016-04-28 17:12:09 +01001928 /* Now we've corralled the various formats, what'll it do? */
Robin Murphy7602b872016-04-28 17:12:09 +01001929 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
Robin Murphyd5466352016-05-09 17:20:09 +01001930 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
Robin Murphy7602b872016-04-28 17:12:09 +01001931 if (smmu->features &
1932 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
Robin Murphyd5466352016-05-09 17:20:09 +01001933 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
Robin Murphy7602b872016-04-28 17:12:09 +01001934 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
Robin Murphyd5466352016-05-09 17:20:09 +01001935 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
Robin Murphy7602b872016-04-28 17:12:09 +01001936 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
Robin Murphyd5466352016-05-09 17:20:09 +01001937 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
Robin Murphy7602b872016-04-28 17:12:09 +01001938
Robin Murphyd5466352016-05-09 17:20:09 +01001939 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1940 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1941 else
1942 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1943 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1944 smmu->pgsize_bitmap);
1945
Will Deacon518f7132014-11-14 17:17:54 +00001946
Will Deacon28d60072014-09-01 16:24:48 +01001947 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1948 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
Will Deacon518f7132014-11-14 17:17:54 +00001949 smmu->va_size, smmu->ipa_size);
Will Deacon28d60072014-09-01 16:24:48 +01001950
1951 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1952 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
Will Deacon518f7132014-11-14 17:17:54 +00001953 smmu->ipa_size, smmu->pa_size);
Will Deacon28d60072014-09-01 16:24:48 +01001954
Will Deacon45ae7cf2013-06-24 18:31:25 +01001955 return 0;
1956}
1957
Robin Murphy67b65a32016-04-13 18:12:57 +01001958struct arm_smmu_match_data {
1959 enum arm_smmu_arch_version version;
1960 enum arm_smmu_implementation model;
1961};
1962
1963#define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1964static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1965
1966ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1967ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
Robin Murphyb7862e32016-04-13 18:13:03 +01001968ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
Robin Murphyf0cfffc2016-04-13 18:12:59 +01001969ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
Robin Murphye086d912016-04-13 18:12:58 +01001970ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
Robin Murphy67b65a32016-04-13 18:12:57 +01001971
Joerg Roedel09b52692014-10-02 12:24:45 +02001972static const struct of_device_id arm_smmu_of_match[] = {
Robin Murphy67b65a32016-04-13 18:12:57 +01001973 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1974 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1975 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
Robin Murphyb7862e32016-04-13 18:13:03 +01001976 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
Robin Murphyf0cfffc2016-04-13 18:12:59 +01001977 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
Robin Murphye086d912016-04-13 18:12:58 +01001978 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
Robin Murphy09360402014-08-28 17:51:59 +01001979 { },
1980};
1981MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1982
Lorenzo Pieralisid6fcd3b2016-11-21 10:01:45 +00001983#ifdef CONFIG_ACPI
1984static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1985{
1986 int ret = 0;
1987
1988 switch (model) {
1989 case ACPI_IORT_SMMU_V1:
1990 case ACPI_IORT_SMMU_CORELINK_MMU400:
1991 smmu->version = ARM_SMMU_V1;
1992 smmu->model = GENERIC_SMMU;
1993 break;
1994 case ACPI_IORT_SMMU_V2:
1995 smmu->version = ARM_SMMU_V2;
1996 smmu->model = GENERIC_SMMU;
1997 break;
1998 case ACPI_IORT_SMMU_CORELINK_MMU500:
1999 smmu->version = ARM_SMMU_V2;
2000 smmu->model = ARM_MMU500;
2001 break;
2002 default:
2003 ret = -ENODEV;
2004 }
2005
2006 return ret;
2007}
2008
2009static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2010 struct arm_smmu_device *smmu)
2011{
2012 struct device *dev = smmu->dev;
2013 struct acpi_iort_node *node =
2014 *(struct acpi_iort_node **)dev_get_platdata(dev);
2015 struct acpi_iort_smmu *iort_smmu;
2016 int ret;
2017
2018 /* Retrieve SMMU1/2 specific data */
2019 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2020
2021 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2022 if (ret < 0)
2023 return ret;
2024
2025 /* Ignore the configuration access interrupt */
2026 smmu->num_global_irqs = 1;
2027
2028 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2029 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2030
2031 return 0;
2032}
2033#else
2034static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2035 struct arm_smmu_device *smmu)
2036{
2037 return -ENODEV;
2038}
2039#endif
2040
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002041static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2042 struct arm_smmu_device *smmu)
Will Deacon45ae7cf2013-06-24 18:31:25 +01002043{
Robin Murphy67b65a32016-04-13 18:12:57 +01002044 const struct arm_smmu_match_data *data;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002045 struct device *dev = &pdev->dev;
Robin Murphy021bb842016-09-14 15:26:46 +01002046 bool legacy_binding;
2047
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002048 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2049 &smmu->num_global_irqs)) {
2050 dev_err(dev, "missing #global-interrupts property\n");
2051 return -ENODEV;
2052 }
2053
2054 data = of_device_get_match_data(dev);
2055 smmu->version = data->version;
2056 smmu->model = data->model;
2057
2058 parse_driver_options(smmu);
2059
Robin Murphy021bb842016-09-14 15:26:46 +01002060 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2061 if (legacy_binding && !using_generic_binding) {
2062 if (!using_legacy_binding)
2063 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2064 using_legacy_binding = true;
2065 } else if (!legacy_binding && !using_legacy_binding) {
2066 using_generic_binding = true;
2067 } else {
2068 dev_err(dev, "not probing due to mismatched DT properties\n");
2069 return -ENODEV;
2070 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01002071
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002072 if (of_dma_is_coherent(dev->of_node))
2073 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2074
2075 return 0;
2076}
2077
2078static int arm_smmu_device_probe(struct platform_device *pdev)
2079{
2080 struct resource *res;
Joerg Roedel9648cbc2017-02-01 18:11:36 +01002081 resource_size_t ioaddr;
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002082 struct arm_smmu_device *smmu;
2083 struct device *dev = &pdev->dev;
2084 int num_irqs, i, err;
2085
Will Deacon45ae7cf2013-06-24 18:31:25 +01002086 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2087 if (!smmu) {
2088 dev_err(dev, "failed to allocate arm_smmu_device\n");
2089 return -ENOMEM;
2090 }
2091 smmu->dev = dev;
2092
Lorenzo Pieralisid6fcd3b2016-11-21 10:01:45 +00002093 if (dev->of_node)
2094 err = arm_smmu_device_dt_probe(pdev, smmu);
2095 else
2096 err = arm_smmu_device_acpi_probe(pdev, smmu);
2097
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002098 if (err)
2099 return err;
Robin Murphy09360402014-08-28 17:51:59 +01002100
Will Deacon45ae7cf2013-06-24 18:31:25 +01002101 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Joerg Roedel9648cbc2017-02-01 18:11:36 +01002102 ioaddr = res->start;
Julia Lawall8a7f4312013-08-19 12:20:37 +01002103 smmu->base = devm_ioremap_resource(dev, res);
2104 if (IS_ERR(smmu->base))
2105 return PTR_ERR(smmu->base);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002106 smmu->size = resource_size(res);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002107
Will Deacon45ae7cf2013-06-24 18:31:25 +01002108 num_irqs = 0;
2109 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2110 num_irqs++;
2111 if (num_irqs > smmu->num_global_irqs)
2112 smmu->num_context_irqs++;
2113 }
2114
Andreas Herrmann44a08de2013-10-01 13:39:07 +01002115 if (!smmu->num_context_irqs) {
2116 dev_err(dev, "found %d interrupts but expected at least %d\n",
2117 num_irqs, smmu->num_global_irqs + 1);
2118 return -ENODEV;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002119 }
Will Deacon45ae7cf2013-06-24 18:31:25 +01002120
2121 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2122 GFP_KERNEL);
2123 if (!smmu->irqs) {
2124 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2125 return -ENOMEM;
2126 }
2127
2128 for (i = 0; i < num_irqs; ++i) {
2129 int irq = platform_get_irq(pdev, i);
Mitchel Humpherys29073202014-07-08 09:52:18 -07002130
Will Deacon45ae7cf2013-06-24 18:31:25 +01002131 if (irq < 0) {
2132 dev_err(dev, "failed to get irq index %d\n", i);
2133 return -ENODEV;
2134 }
2135 smmu->irqs[i] = irq;
2136 }
2137
Olav Haugan3c8766d2014-08-22 17:12:32 -07002138 err = arm_smmu_device_cfg_probe(smmu);
2139 if (err)
2140 return err;
2141
Robin Murphyb7862e32016-04-13 18:13:03 +01002142 if (smmu->version == ARM_SMMU_V2 &&
Will Deacon45ae7cf2013-06-24 18:31:25 +01002143 smmu->num_context_banks != smmu->num_context_irqs) {
2144 dev_err(dev,
2145 "found only %d context interrupt(s) but %d required\n",
2146 smmu->num_context_irqs, smmu->num_context_banks);
Robin Murphyf80cd882016-09-14 15:21:39 +01002147 return -ENODEV;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002148 }
2149
Will Deacon45ae7cf2013-06-24 18:31:25 +01002150 for (i = 0; i < smmu->num_global_irqs; ++i) {
Peng Fanbee14002016-07-04 17:38:22 +08002151 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2152 arm_smmu_global_fault,
2153 IRQF_SHARED,
2154 "arm-smmu global fault",
2155 smmu);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002156 if (err) {
2157 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2158 i, smmu->irqs[i]);
Robin Murphyf80cd882016-09-14 15:21:39 +01002159 return err;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002160 }
2161 }
2162
Joerg Roedel9648cbc2017-02-01 18:11:36 +01002163 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2164 "smmu.%pa", &ioaddr);
2165 if (err) {
2166 dev_err(dev, "Failed to register iommu in sysfs\n");
2167 return err;
2168 }
2169
2170 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2171 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2172
2173 err = iommu_device_register(&smmu->iommu);
2174 if (err) {
2175 dev_err(dev, "Failed to register iommu\n");
2176 return err;
2177 }
2178
Robin Murphyd6fc5d92016-09-12 17:13:52 +01002179 platform_set_drvdata(pdev, smmu);
Will Deaconfd90cec2013-08-21 13:56:34 +01002180 arm_smmu_device_reset(smmu);
Aleksey Makarovdc0eaa42017-01-19 17:36:36 +03002181 arm_smmu_test_smr_masks(smmu);
Robin Murphy021bb842016-09-14 15:26:46 +01002182
2183 /* Oh, for a proper bus abstraction */
2184 if (!iommu_present(&platform_bus_type))
2185 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2186#ifdef CONFIG_ARM_AMBA
2187 if (!iommu_present(&amba_bustype))
2188 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2189#endif
2190#ifdef CONFIG_PCI
2191 if (!iommu_present(&pci_bus_type)) {
2192 pci_request_acs();
2193 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2194 }
2195#endif
Will Deacon45ae7cf2013-06-24 18:31:25 +01002196 return 0;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002197}
2198
2199static int arm_smmu_device_remove(struct platform_device *pdev)
2200{
Robin Murphyd6fc5d92016-09-12 17:13:52 +01002201 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002202
2203 if (!smmu)
2204 return -ENODEV;
2205
Will Deaconecfadb62013-07-31 19:21:28 +01002206 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
Robin Murphyd6fc5d92016-09-12 17:13:52 +01002207 dev_err(&pdev->dev, "removing device with active domains!\n");
Will Deacon45ae7cf2013-06-24 18:31:25 +01002208
Will Deacon45ae7cf2013-06-24 18:31:25 +01002209 /* Turn the thing off */
Mitchel Humpherys29073202014-07-08 09:52:18 -07002210 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002211 return 0;
2212}
2213
Will Deacon45ae7cf2013-06-24 18:31:25 +01002214static struct platform_driver arm_smmu_driver = {
2215 .driver = {
Will Deacon45ae7cf2013-06-24 18:31:25 +01002216 .name = "arm-smmu",
2217 .of_match_table = of_match_ptr(arm_smmu_of_match),
2218 },
Lorenzo Pieralisibbb8a182016-11-21 10:01:44 +00002219 .probe = arm_smmu_device_probe,
Will Deacon45ae7cf2013-06-24 18:31:25 +01002220 .remove = arm_smmu_device_remove,
2221};
2222
2223static int __init arm_smmu_init(void)
2224{
Robin Murphy021bb842016-09-14 15:26:46 +01002225 static bool registered;
2226 int ret = 0;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002227
Robin Murphy021bb842016-09-14 15:26:46 +01002228 if (!registered) {
2229 ret = platform_driver_register(&arm_smmu_driver);
2230 registered = !ret;
Wei Chen112c8982016-06-13 17:20:17 +08002231 }
Robin Murphy021bb842016-09-14 15:26:46 +01002232 return ret;
Will Deacon45ae7cf2013-06-24 18:31:25 +01002233}
2234
2235static void __exit arm_smmu_exit(void)
2236{
2237 return platform_driver_unregister(&arm_smmu_driver);
2238}
2239
Andreas Herrmannb1950b22013-10-01 13:39:05 +01002240subsys_initcall(arm_smmu_init);
Will Deacon45ae7cf2013-06-24 18:31:25 +01002241module_exit(arm_smmu_exit);
2242
Robin Murphy021bb842016-09-14 15:26:46 +01002243static int __init arm_smmu_of_init(struct device_node *np)
2244{
2245 int ret = arm_smmu_init();
2246
2247 if (ret)
2248 return ret;
2249
2250 if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2251 return -ENODEV;
2252
2253 return 0;
2254}
2255IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2256IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2257IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2258IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2259IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2260IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2261
Lorenzo Pieralisid6fcd3b2016-11-21 10:01:45 +00002262#ifdef CONFIG_ACPI
2263static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2264{
2265 if (iort_node_match(ACPI_IORT_NODE_SMMU))
2266 return arm_smmu_init();
2267
2268 return 0;
2269}
2270IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2271#endif
2272
Will Deacon45ae7cf2013-06-24 18:31:25 +01002273MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2274MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2275MODULE_LICENSE("GPL v2");