blob: 20875341c865e20512afe2c44b2422181dd0f521 [file] [log] [blame]
Will Deacon48ec83b2015-05-27 17:25:59 +01001/*
2 * IOMMU API for ARM architected SMMUv3 implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2015 ARM Limited
17 *
18 * Author: Will Deacon <will.deacon@arm.com>
19 *
20 * This driver is powered by bad coffee and bombay mix.
21 */
22
23#include <linux/delay.h>
24#include <linux/err.h>
25#include <linux/interrupt.h>
26#include <linux/iommu.h>
27#include <linux/iopoll.h>
28#include <linux/module.h>
Marc Zyngier166bdbd2015-10-13 18:32:30 +010029#include <linux/msi.h>
Will Deacon48ec83b2015-05-27 17:25:59 +010030#include <linux/of.h>
31#include <linux/of_address.h>
Will Deacon941a8022015-08-11 16:25:10 +010032#include <linux/of_platform.h>
Will Deacon48ec83b2015-05-27 17:25:59 +010033#include <linux/pci.h>
34#include <linux/platform_device.h>
35
36#include "io-pgtable.h"
37
38/* MMIO registers */
39#define ARM_SMMU_IDR0 0x0
40#define IDR0_ST_LVL_SHIFT 27
41#define IDR0_ST_LVL_MASK 0x3
42#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
Prem Mallappa6380be02015-12-14 22:01:23 +053043#define IDR0_STALL_MODEL_SHIFT 24
44#define IDR0_STALL_MODEL_MASK 0x3
45#define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT)
46#define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT)
Will Deacon48ec83b2015-05-27 17:25:59 +010047#define IDR0_TTENDIAN_SHIFT 21
48#define IDR0_TTENDIAN_MASK 0x3
49#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
50#define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
51#define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
52#define IDR0_CD2L (1 << 19)
53#define IDR0_VMID16 (1 << 18)
54#define IDR0_PRI (1 << 16)
55#define IDR0_SEV (1 << 14)
56#define IDR0_MSI (1 << 13)
57#define IDR0_ASID16 (1 << 12)
58#define IDR0_ATS (1 << 10)
59#define IDR0_HYP (1 << 9)
60#define IDR0_COHACC (1 << 4)
61#define IDR0_TTF_SHIFT 2
62#define IDR0_TTF_MASK 0x3
63#define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
Will Deaconf0c453d2015-08-20 12:12:32 +010064#define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
Will Deacon48ec83b2015-05-27 17:25:59 +010065#define IDR0_S1P (1 << 1)
66#define IDR0_S2P (1 << 0)
67
68#define ARM_SMMU_IDR1 0x4
69#define IDR1_TABLES_PRESET (1 << 30)
70#define IDR1_QUEUES_PRESET (1 << 29)
71#define IDR1_REL (1 << 28)
72#define IDR1_CMDQ_SHIFT 21
73#define IDR1_CMDQ_MASK 0x1f
74#define IDR1_EVTQ_SHIFT 16
75#define IDR1_EVTQ_MASK 0x1f
76#define IDR1_PRIQ_SHIFT 11
77#define IDR1_PRIQ_MASK 0x1f
78#define IDR1_SSID_SHIFT 6
79#define IDR1_SSID_MASK 0x1f
80#define IDR1_SID_SHIFT 0
81#define IDR1_SID_MASK 0x3f
82
83#define ARM_SMMU_IDR5 0x14
84#define IDR5_STALL_MAX_SHIFT 16
85#define IDR5_STALL_MAX_MASK 0xffff
86#define IDR5_GRAN64K (1 << 6)
87#define IDR5_GRAN16K (1 << 5)
88#define IDR5_GRAN4K (1 << 4)
89#define IDR5_OAS_SHIFT 0
90#define IDR5_OAS_MASK 0x7
91#define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
92#define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
93#define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
94#define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
95#define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
96#define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
97
98#define ARM_SMMU_CR0 0x20
99#define CR0_CMDQEN (1 << 3)
100#define CR0_EVTQEN (1 << 2)
101#define CR0_PRIQEN (1 << 1)
102#define CR0_SMMUEN (1 << 0)
103
104#define ARM_SMMU_CR0ACK 0x24
105
106#define ARM_SMMU_CR1 0x28
107#define CR1_SH_NSH 0
108#define CR1_SH_OSH 2
109#define CR1_SH_ISH 3
110#define CR1_CACHE_NC 0
111#define CR1_CACHE_WB 1
112#define CR1_CACHE_WT 2
113#define CR1_TABLE_SH_SHIFT 10
114#define CR1_TABLE_OC_SHIFT 8
115#define CR1_TABLE_IC_SHIFT 6
116#define CR1_QUEUE_SH_SHIFT 4
117#define CR1_QUEUE_OC_SHIFT 2
118#define CR1_QUEUE_IC_SHIFT 0
119
120#define ARM_SMMU_CR2 0x2c
121#define CR2_PTM (1 << 2)
122#define CR2_RECINVSID (1 << 1)
123#define CR2_E2H (1 << 0)
124
125#define ARM_SMMU_IRQ_CTRL 0x50
126#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
Marc Zyngierccd63852015-07-15 11:55:18 +0100127#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
Will Deacon48ec83b2015-05-27 17:25:59 +0100128#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
129
130#define ARM_SMMU_IRQ_CTRLACK 0x54
131
132#define ARM_SMMU_GERROR 0x60
133#define GERROR_SFM_ERR (1 << 8)
134#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
135#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
136#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
137#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
138#define GERROR_PRIQ_ABT_ERR (1 << 3)
139#define GERROR_EVTQ_ABT_ERR (1 << 2)
140#define GERROR_CMDQ_ERR (1 << 0)
141#define GERROR_ERR_MASK 0xfd
142
143#define ARM_SMMU_GERRORN 0x64
144
145#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
146#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
147#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
148
149#define ARM_SMMU_STRTAB_BASE 0x80
150#define STRTAB_BASE_RA (1UL << 62)
151#define STRTAB_BASE_ADDR_SHIFT 6
152#define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
153
154#define ARM_SMMU_STRTAB_BASE_CFG 0x88
155#define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
156#define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
157#define STRTAB_BASE_CFG_SPLIT_SHIFT 6
158#define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
159#define STRTAB_BASE_CFG_FMT_SHIFT 16
160#define STRTAB_BASE_CFG_FMT_MASK 0x3
161#define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
162#define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
163
164#define ARM_SMMU_CMDQ_BASE 0x90
165#define ARM_SMMU_CMDQ_PROD 0x98
166#define ARM_SMMU_CMDQ_CONS 0x9c
167
168#define ARM_SMMU_EVTQ_BASE 0xa0
169#define ARM_SMMU_EVTQ_PROD 0x100a8
170#define ARM_SMMU_EVTQ_CONS 0x100ac
171#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
172#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
173#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
174
175#define ARM_SMMU_PRIQ_BASE 0xc0
176#define ARM_SMMU_PRIQ_PROD 0x100c8
177#define ARM_SMMU_PRIQ_CONS 0x100cc
178#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
179#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
180#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
181
182/* Common MSI config fields */
Will Deacon48ec83b2015-05-27 17:25:59 +0100183#define MSI_CFG0_ADDR_SHIFT 2
184#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
Marc Zyngierec11d632015-07-15 11:55:19 +0100185#define MSI_CFG2_SH_SHIFT 4
186#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
187#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
188#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
189#define MSI_CFG2_MEMATTR_SHIFT 0
190#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
Will Deacon48ec83b2015-05-27 17:25:59 +0100191
192#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
193#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
194#define Q_OVERFLOW_FLAG (1 << 31)
195#define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
196#define Q_ENT(q, p) ((q)->base + \
197 Q_IDX(q, p) * (q)->ent_dwords)
198
199#define Q_BASE_RWA (1UL << 62)
200#define Q_BASE_ADDR_SHIFT 5
201#define Q_BASE_ADDR_MASK 0xfffffffffffUL
202#define Q_BASE_LOG2SIZE_SHIFT 0
203#define Q_BASE_LOG2SIZE_MASK 0x1fUL
204
205/*
206 * Stream table.
207 *
208 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
Zhen Leie2f4c232015-07-07 04:30:17 +0100209 * 2lvl: 128k L1 entries,
210 * 256 lazy entries per table (each table covers a PCI bus)
Will Deacon48ec83b2015-05-27 17:25:59 +0100211 */
Zhen Leie2f4c232015-07-07 04:30:17 +0100212#define STRTAB_L1_SZ_SHIFT 20
Will Deacon48ec83b2015-05-27 17:25:59 +0100213#define STRTAB_SPLIT 8
214
215#define STRTAB_L1_DESC_DWORDS 1
216#define STRTAB_L1_DESC_SPAN_SHIFT 0
217#define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
218#define STRTAB_L1_DESC_L2PTR_SHIFT 6
219#define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
220
221#define STRTAB_STE_DWORDS 8
222#define STRTAB_STE_0_V (1UL << 0)
223#define STRTAB_STE_0_CFG_SHIFT 1
224#define STRTAB_STE_0_CFG_MASK 0x7UL
225#define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
226#define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
227#define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
228#define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
229
230#define STRTAB_STE_0_S1FMT_SHIFT 4
231#define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
232#define STRTAB_STE_0_S1CTXPTR_SHIFT 6
233#define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
234#define STRTAB_STE_0_S1CDMAX_SHIFT 59
235#define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
236
237#define STRTAB_STE_1_S1C_CACHE_NC 0UL
238#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
239#define STRTAB_STE_1_S1C_CACHE_WT 2UL
240#define STRTAB_STE_1_S1C_CACHE_WB 3UL
241#define STRTAB_STE_1_S1C_SH_NSH 0UL
242#define STRTAB_STE_1_S1C_SH_OSH 2UL
243#define STRTAB_STE_1_S1C_SH_ISH 3UL
244#define STRTAB_STE_1_S1CIR_SHIFT 2
245#define STRTAB_STE_1_S1COR_SHIFT 4
246#define STRTAB_STE_1_S1CSH_SHIFT 6
247
248#define STRTAB_STE_1_S1STALLD (1UL << 27)
249
250#define STRTAB_STE_1_EATS_ABT 0UL
251#define STRTAB_STE_1_EATS_TRANS 1UL
252#define STRTAB_STE_1_EATS_S1CHK 2UL
253#define STRTAB_STE_1_EATS_SHIFT 28
254
255#define STRTAB_STE_1_STRW_NSEL1 0UL
256#define STRTAB_STE_1_STRW_EL2 2UL
257#define STRTAB_STE_1_STRW_SHIFT 30
258
Will Deacona0eacd82015-11-18 18:15:51 +0000259#define STRTAB_STE_1_SHCFG_INCOMING 1UL
260#define STRTAB_STE_1_SHCFG_SHIFT 44
261
Will Deacon48ec83b2015-05-27 17:25:59 +0100262#define STRTAB_STE_2_S2VMID_SHIFT 0
263#define STRTAB_STE_2_S2VMID_MASK 0xffffUL
264#define STRTAB_STE_2_VTCR_SHIFT 32
265#define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
266#define STRTAB_STE_2_S2AA64 (1UL << 51)
267#define STRTAB_STE_2_S2ENDI (1UL << 52)
268#define STRTAB_STE_2_S2PTW (1UL << 54)
269#define STRTAB_STE_2_S2R (1UL << 58)
270
271#define STRTAB_STE_3_S2TTB_SHIFT 4
272#define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
273
274/* Context descriptor (stage-1 only) */
275#define CTXDESC_CD_DWORDS 8
276#define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
277#define ARM64_TCR_T0SZ_SHIFT 0
278#define ARM64_TCR_T0SZ_MASK 0x1fUL
279#define CTXDESC_CD_0_TCR_TG0_SHIFT 6
280#define ARM64_TCR_TG0_SHIFT 14
281#define ARM64_TCR_TG0_MASK 0x3UL
282#define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
Zhen Lei5d58c622015-06-26 09:32:59 +0100283#define ARM64_TCR_IRGN0_SHIFT 8
Will Deacon48ec83b2015-05-27 17:25:59 +0100284#define ARM64_TCR_IRGN0_MASK 0x3UL
285#define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
Zhen Lei5d58c622015-06-26 09:32:59 +0100286#define ARM64_TCR_ORGN0_SHIFT 10
Will Deacon48ec83b2015-05-27 17:25:59 +0100287#define ARM64_TCR_ORGN0_MASK 0x3UL
288#define CTXDESC_CD_0_TCR_SH0_SHIFT 12
289#define ARM64_TCR_SH0_SHIFT 12
290#define ARM64_TCR_SH0_MASK 0x3UL
291#define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
292#define ARM64_TCR_EPD0_SHIFT 7
293#define ARM64_TCR_EPD0_MASK 0x1UL
294#define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
295#define ARM64_TCR_EPD1_SHIFT 23
296#define ARM64_TCR_EPD1_MASK 0x1UL
297
298#define CTXDESC_CD_0_ENDI (1UL << 15)
299#define CTXDESC_CD_0_V (1UL << 31)
300
301#define CTXDESC_CD_0_TCR_IPS_SHIFT 32
302#define ARM64_TCR_IPS_SHIFT 32
303#define ARM64_TCR_IPS_MASK 0x7UL
304#define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
305#define ARM64_TCR_TBI0_SHIFT 37
306#define ARM64_TCR_TBI0_MASK 0x1UL
307
308#define CTXDESC_CD_0_AA64 (1UL << 41)
309#define CTXDESC_CD_0_R (1UL << 45)
310#define CTXDESC_CD_0_A (1UL << 46)
311#define CTXDESC_CD_0_ASET_SHIFT 47
312#define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
313#define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
314#define CTXDESC_CD_0_ASID_SHIFT 48
315#define CTXDESC_CD_0_ASID_MASK 0xffffUL
316
317#define CTXDESC_CD_1_TTB0_SHIFT 4
318#define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
319
320#define CTXDESC_CD_3_MAIR_SHIFT 0
321
322/* Convert between AArch64 (CPU) TCR format and SMMU CD format */
323#define ARM_SMMU_TCR2CD(tcr, fld) \
324 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
325 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
326
327/* Command queue */
328#define CMDQ_ENT_DWORDS 2
329#define CMDQ_MAX_SZ_SHIFT 8
330
331#define CMDQ_ERR_SHIFT 24
332#define CMDQ_ERR_MASK 0x7f
333#define CMDQ_ERR_CERROR_NONE_IDX 0
334#define CMDQ_ERR_CERROR_ILL_IDX 1
335#define CMDQ_ERR_CERROR_ABT_IDX 2
336
337#define CMDQ_0_OP_SHIFT 0
338#define CMDQ_0_OP_MASK 0xffUL
339#define CMDQ_0_SSV (1UL << 11)
340
341#define CMDQ_PREFETCH_0_SID_SHIFT 32
342#define CMDQ_PREFETCH_1_SIZE_SHIFT 0
343#define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
344
345#define CMDQ_CFGI_0_SID_SHIFT 32
346#define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
347#define CMDQ_CFGI_1_LEAF (1UL << 0)
348#define CMDQ_CFGI_1_RANGE_SHIFT 0
349#define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
350
351#define CMDQ_TLBI_0_VMID_SHIFT 32
352#define CMDQ_TLBI_0_ASID_SHIFT 48
353#define CMDQ_TLBI_1_LEAF (1UL << 0)
Will Deacon1c27df12015-09-18 16:12:56 +0100354#define CMDQ_TLBI_1_VA_MASK ~0xfffUL
355#define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
Will Deacon48ec83b2015-05-27 17:25:59 +0100356
357#define CMDQ_PRI_0_SSID_SHIFT 12
358#define CMDQ_PRI_0_SSID_MASK 0xfffffUL
359#define CMDQ_PRI_0_SID_SHIFT 32
360#define CMDQ_PRI_0_SID_MASK 0xffffffffUL
361#define CMDQ_PRI_1_GRPID_SHIFT 0
362#define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
363#define CMDQ_PRI_1_RESP_SHIFT 12
364#define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
365#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
366#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
367
368#define CMDQ_SYNC_0_CS_SHIFT 12
369#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
370#define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
371
372/* Event queue */
373#define EVTQ_ENT_DWORDS 4
374#define EVTQ_MAX_SZ_SHIFT 7
375
376#define EVTQ_0_ID_SHIFT 0
377#define EVTQ_0_ID_MASK 0xffUL
378
379/* PRI queue */
380#define PRIQ_ENT_DWORDS 2
381#define PRIQ_MAX_SZ_SHIFT 8
382
383#define PRIQ_0_SID_SHIFT 0
384#define PRIQ_0_SID_MASK 0xffffffffUL
385#define PRIQ_0_SSID_SHIFT 32
386#define PRIQ_0_SSID_MASK 0xfffffUL
Will Deacon48ec83b2015-05-27 17:25:59 +0100387#define PRIQ_0_PERM_PRIV (1UL << 58)
388#define PRIQ_0_PERM_EXEC (1UL << 59)
389#define PRIQ_0_PERM_READ (1UL << 60)
390#define PRIQ_0_PERM_WRITE (1UL << 61)
391#define PRIQ_0_PRG_LAST (1UL << 62)
392#define PRIQ_0_SSID_V (1UL << 63)
393
394#define PRIQ_1_PRG_IDX_SHIFT 0
395#define PRIQ_1_PRG_IDX_MASK 0x1ffUL
396#define PRIQ_1_ADDR_SHIFT 12
397#define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
398
399/* High-level queue structures */
400#define ARM_SMMU_POLL_TIMEOUT_US 100
401
402static bool disable_bypass;
403module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
404MODULE_PARM_DESC(disable_bypass,
405 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
406
407enum pri_resp {
408 PRI_RESP_DENY,
409 PRI_RESP_FAIL,
410 PRI_RESP_SUCC,
411};
412
Marc Zyngier166bdbd2015-10-13 18:32:30 +0100413enum arm_smmu_msi_index {
414 EVTQ_MSI_INDEX,
415 GERROR_MSI_INDEX,
416 PRIQ_MSI_INDEX,
417 ARM_SMMU_MAX_MSIS,
418};
419
420static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
421 [EVTQ_MSI_INDEX] = {
422 ARM_SMMU_EVTQ_IRQ_CFG0,
423 ARM_SMMU_EVTQ_IRQ_CFG1,
424 ARM_SMMU_EVTQ_IRQ_CFG2,
425 },
426 [GERROR_MSI_INDEX] = {
427 ARM_SMMU_GERROR_IRQ_CFG0,
428 ARM_SMMU_GERROR_IRQ_CFG1,
429 ARM_SMMU_GERROR_IRQ_CFG2,
430 },
431 [PRIQ_MSI_INDEX] = {
432 ARM_SMMU_PRIQ_IRQ_CFG0,
433 ARM_SMMU_PRIQ_IRQ_CFG1,
434 ARM_SMMU_PRIQ_IRQ_CFG2,
435 },
436};
437
Will Deacon48ec83b2015-05-27 17:25:59 +0100438struct arm_smmu_cmdq_ent {
439 /* Common fields */
440 u8 opcode;
441 bool substream_valid;
442
443 /* Command-specific fields */
444 union {
445 #define CMDQ_OP_PREFETCH_CFG 0x1
446 struct {
447 u32 sid;
448 u8 size;
449 u64 addr;
450 } prefetch;
451
452 #define CMDQ_OP_CFGI_STE 0x3
453 #define CMDQ_OP_CFGI_ALL 0x4
454 struct {
455 u32 sid;
456 union {
457 bool leaf;
458 u8 span;
459 };
460 } cfgi;
461
462 #define CMDQ_OP_TLBI_NH_ASID 0x11
463 #define CMDQ_OP_TLBI_NH_VA 0x12
464 #define CMDQ_OP_TLBI_EL2_ALL 0x20
465 #define CMDQ_OP_TLBI_S12_VMALL 0x28
466 #define CMDQ_OP_TLBI_S2_IPA 0x2a
467 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
468 struct {
469 u16 asid;
470 u16 vmid;
471 bool leaf;
472 u64 addr;
473 } tlbi;
474
475 #define CMDQ_OP_PRI_RESP 0x41
476 struct {
477 u32 sid;
478 u32 ssid;
479 u16 grpid;
480 enum pri_resp resp;
481 } pri;
482
483 #define CMDQ_OP_CMD_SYNC 0x46
484 };
485};
486
487struct arm_smmu_queue {
488 int irq; /* Wired interrupt */
489
490 __le64 *base;
491 dma_addr_t base_dma;
492 u64 q_base;
493
494 size_t ent_dwords;
495 u32 max_n_shift;
496 u32 prod;
497 u32 cons;
498
499 u32 __iomem *prod_reg;
500 u32 __iomem *cons_reg;
501};
502
503struct arm_smmu_cmdq {
504 struct arm_smmu_queue q;
505 spinlock_t lock;
506};
507
508struct arm_smmu_evtq {
509 struct arm_smmu_queue q;
510 u32 max_stalls;
511};
512
513struct arm_smmu_priq {
514 struct arm_smmu_queue q;
515};
516
517/* High-level stream table and context descriptor structures */
518struct arm_smmu_strtab_l1_desc {
519 u8 span;
520
521 __le64 *l2ptr;
522 dma_addr_t l2ptr_dma;
523};
524
525struct arm_smmu_s1_cfg {
526 __le64 *cdptr;
527 dma_addr_t cdptr_dma;
528
529 struct arm_smmu_ctx_desc {
530 u16 asid;
531 u64 ttbr;
532 u64 tcr;
533 u64 mair;
534 } cd;
535};
536
537struct arm_smmu_s2_cfg {
538 u16 vmid;
539 u64 vttbr;
540 u64 vtcr;
541};
542
543struct arm_smmu_strtab_ent {
544 bool valid;
545
546 bool bypass; /* Overrides s1/s2 config */
547 struct arm_smmu_s1_cfg *s1_cfg;
548 struct arm_smmu_s2_cfg *s2_cfg;
549};
550
551struct arm_smmu_strtab_cfg {
552 __le64 *strtab;
553 dma_addr_t strtab_dma;
554 struct arm_smmu_strtab_l1_desc *l1_desc;
555 unsigned int num_l1_ents;
556
557 u64 strtab_base;
558 u32 strtab_base_cfg;
559};
560
561/* An SMMUv3 instance */
562struct arm_smmu_device {
563 struct device *dev;
564 void __iomem *base;
565
566#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
567#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
568#define ARM_SMMU_FEAT_TT_LE (1 << 2)
569#define ARM_SMMU_FEAT_TT_BE (1 << 3)
570#define ARM_SMMU_FEAT_PRI (1 << 4)
571#define ARM_SMMU_FEAT_ATS (1 << 5)
572#define ARM_SMMU_FEAT_SEV (1 << 6)
573#define ARM_SMMU_FEAT_MSI (1 << 7)
574#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
575#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
576#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
577#define ARM_SMMU_FEAT_STALLS (1 << 11)
578#define ARM_SMMU_FEAT_HYP (1 << 12)
579 u32 features;
580
Zhen Lei5e929462015-07-07 04:30:18 +0100581#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
582 u32 options;
583
Will Deacon48ec83b2015-05-27 17:25:59 +0100584 struct arm_smmu_cmdq cmdq;
585 struct arm_smmu_evtq evtq;
586 struct arm_smmu_priq priq;
587
588 int gerr_irq;
589
590 unsigned long ias; /* IPA */
591 unsigned long oas; /* PA */
592
593#define ARM_SMMU_MAX_ASIDS (1 << 16)
594 unsigned int asid_bits;
595 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
596
597#define ARM_SMMU_MAX_VMIDS (1 << 16)
598 unsigned int vmid_bits;
599 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
600
601 unsigned int ssid_bits;
602 unsigned int sid_bits;
603
604 struct arm_smmu_strtab_cfg strtab_cfg;
Will Deacon48ec83b2015-05-27 17:25:59 +0100605};
606
607/* SMMU private data for an IOMMU group */
608struct arm_smmu_group {
609 struct arm_smmu_device *smmu;
610 struct arm_smmu_domain *domain;
611 int num_sids;
612 u32 *sids;
613 struct arm_smmu_strtab_ent ste;
614};
615
616/* SMMU private data for an IOMMU domain */
617enum arm_smmu_domain_stage {
618 ARM_SMMU_DOMAIN_S1 = 0,
619 ARM_SMMU_DOMAIN_S2,
620 ARM_SMMU_DOMAIN_NESTED,
621};
622
623struct arm_smmu_domain {
624 struct arm_smmu_device *smmu;
625 struct mutex init_mutex; /* Protects smmu pointer */
626
627 struct io_pgtable_ops *pgtbl_ops;
628 spinlock_t pgtbl_lock;
629
630 enum arm_smmu_domain_stage stage;
631 union {
632 struct arm_smmu_s1_cfg s1_cfg;
633 struct arm_smmu_s2_cfg s2_cfg;
634 };
635
636 struct iommu_domain domain;
637};
638
Zhen Lei5e929462015-07-07 04:30:18 +0100639struct arm_smmu_option_prop {
640 u32 opt;
641 const char *prop;
642};
643
644static struct arm_smmu_option_prop arm_smmu_options[] = {
645 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
646 { 0, NULL},
647};
648
Will Deacon48ec83b2015-05-27 17:25:59 +0100649static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
650{
651 return container_of(dom, struct arm_smmu_domain, domain);
652}
653
Zhen Lei5e929462015-07-07 04:30:18 +0100654static void parse_driver_options(struct arm_smmu_device *smmu)
655{
656 int i = 0;
657
658 do {
659 if (of_property_read_bool(smmu->dev->of_node,
660 arm_smmu_options[i].prop)) {
661 smmu->options |= arm_smmu_options[i].opt;
662 dev_notice(smmu->dev, "option %s\n",
663 arm_smmu_options[i].prop);
664 }
665 } while (arm_smmu_options[++i].opt);
666}
667
Will Deacon48ec83b2015-05-27 17:25:59 +0100668/* Low-level queue manipulation functions */
669static bool queue_full(struct arm_smmu_queue *q)
670{
671 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
672 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
673}
674
675static bool queue_empty(struct arm_smmu_queue *q)
676{
677 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
678 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
679}
680
681static void queue_sync_cons(struct arm_smmu_queue *q)
682{
683 q->cons = readl_relaxed(q->cons_reg);
684}
685
686static void queue_inc_cons(struct arm_smmu_queue *q)
687{
688 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
689
690 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
691 writel(q->cons, q->cons_reg);
692}
693
694static int queue_sync_prod(struct arm_smmu_queue *q)
695{
696 int ret = 0;
697 u32 prod = readl_relaxed(q->prod_reg);
698
699 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
700 ret = -EOVERFLOW;
701
702 q->prod = prod;
703 return ret;
704}
705
706static void queue_inc_prod(struct arm_smmu_queue *q)
707{
708 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
709
710 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
711 writel(q->prod, q->prod_reg);
712}
713
714static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)
715{
716 if (Q_WRP(q, q->cons) == Q_WRP(q, until))
717 return Q_IDX(q, q->cons) < Q_IDX(q, until);
718
719 return Q_IDX(q, q->cons) >= Q_IDX(q, until);
720}
721
722static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)
723{
724 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
725
726 while (queue_sync_cons(q), __queue_cons_before(q, until)) {
727 if (ktime_compare(ktime_get(), timeout) > 0)
728 return -ETIMEDOUT;
729
730 if (wfe) {
731 wfe();
732 } else {
733 cpu_relax();
734 udelay(1);
735 }
736 }
737
738 return 0;
739}
740
741static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
742{
743 int i;
744
745 for (i = 0; i < n_dwords; ++i)
746 *dst++ = cpu_to_le64(*src++);
747}
748
749static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
750{
751 if (queue_full(q))
752 return -ENOSPC;
753
754 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
755 queue_inc_prod(q);
756 return 0;
757}
758
759static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
760{
761 int i;
762
763 for (i = 0; i < n_dwords; ++i)
764 *dst++ = le64_to_cpu(*src++);
765}
766
767static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
768{
769 if (queue_empty(q))
770 return -EAGAIN;
771
772 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
773 queue_inc_cons(q);
774 return 0;
775}
776
777/* High-level queue accessors */
778static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
779{
780 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
781 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
782
783 switch (ent->opcode) {
784 case CMDQ_OP_TLBI_EL2_ALL:
785 case CMDQ_OP_TLBI_NSNH_ALL:
786 break;
787 case CMDQ_OP_PREFETCH_CFG:
788 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
789 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
790 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
791 break;
792 case CMDQ_OP_CFGI_STE:
793 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
794 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
795 break;
796 case CMDQ_OP_CFGI_ALL:
797 /* Cover the entire SID range */
798 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
799 break;
800 case CMDQ_OP_TLBI_NH_VA:
801 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
Will Deacon1c27df12015-09-18 16:12:56 +0100802 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
803 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
804 break;
Will Deacon48ec83b2015-05-27 17:25:59 +0100805 case CMDQ_OP_TLBI_S2_IPA:
806 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
807 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
Will Deacon1c27df12015-09-18 16:12:56 +0100808 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
Will Deacon48ec83b2015-05-27 17:25:59 +0100809 break;
810 case CMDQ_OP_TLBI_NH_ASID:
811 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
812 /* Fallthrough */
813 case CMDQ_OP_TLBI_S12_VMALL:
814 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
815 break;
816 case CMDQ_OP_PRI_RESP:
817 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
818 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
819 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
820 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
821 switch (ent->pri.resp) {
822 case PRI_RESP_DENY:
823 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
824 break;
825 case PRI_RESP_FAIL:
826 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
827 break;
828 case PRI_RESP_SUCC:
829 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
830 break;
831 default:
832 return -EINVAL;
833 }
834 break;
835 case CMDQ_OP_CMD_SYNC:
836 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
837 break;
838 default:
839 return -ENOENT;
840 }
841
842 return 0;
843}
844
845static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
846{
847 static const char *cerror_str[] = {
848 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
849 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
850 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
851 };
852
853 int i;
854 u64 cmd[CMDQ_ENT_DWORDS];
855 struct arm_smmu_queue *q = &smmu->cmdq.q;
856 u32 cons = readl_relaxed(q->cons_reg);
857 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
858 struct arm_smmu_cmdq_ent cmd_sync = {
859 .opcode = CMDQ_OP_CMD_SYNC,
860 };
861
862 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
Will Deacona0d5c042015-12-04 12:00:29 +0000863 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
Will Deacon48ec83b2015-05-27 17:25:59 +0100864
865 switch (idx) {
Will Deacon48ec83b2015-05-27 17:25:59 +0100866 case CMDQ_ERR_CERROR_ABT_IDX:
867 dev_err(smmu->dev, "retrying command fetch\n");
868 case CMDQ_ERR_CERROR_NONE_IDX:
869 return;
Will Deacona0d5c042015-12-04 12:00:29 +0000870 case CMDQ_ERR_CERROR_ILL_IDX:
871 /* Fallthrough */
872 default:
873 break;
Will Deacon48ec83b2015-05-27 17:25:59 +0100874 }
875
876 /*
877 * We may have concurrent producers, so we need to be careful
878 * not to touch any of the shadow cmdq state.
879 */
880 queue_read(cmd, Q_ENT(q, idx), q->ent_dwords);
881 dev_err(smmu->dev, "skipping command in error state:\n");
882 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
883 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
884
885 /* Convert the erroneous command into a CMD_SYNC */
886 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
887 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
888 return;
889 }
890
891 queue_write(cmd, Q_ENT(q, idx), q->ent_dwords);
892}
893
894static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
895 struct arm_smmu_cmdq_ent *ent)
896{
897 u32 until;
898 u64 cmd[CMDQ_ENT_DWORDS];
899 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
900 struct arm_smmu_queue *q = &smmu->cmdq.q;
901
902 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
903 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
904 ent->opcode);
905 return;
906 }
907
908 spin_lock(&smmu->cmdq.lock);
909 while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {
910 /*
911 * Keep the queue locked, otherwise the producer could wrap
912 * twice and we could see a future consumer pointer that looks
913 * like it's behind us.
914 */
915 if (queue_poll_cons(q, until, wfe))
916 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
917 }
918
919 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))
920 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
921 spin_unlock(&smmu->cmdq.lock);
922}
923
924/* Context descriptor manipulation functions */
925static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
926{
927 u64 val = 0;
928
929 /* Repack the TCR. Just care about TTBR0 for now */
930 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
931 val |= ARM_SMMU_TCR2CD(tcr, TG0);
932 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
933 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
934 val |= ARM_SMMU_TCR2CD(tcr, SH0);
935 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
936 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
937 val |= ARM_SMMU_TCR2CD(tcr, IPS);
938 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
939
940 return val;
941}
942
943static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
944 struct arm_smmu_s1_cfg *cfg)
945{
946 u64 val;
947
948 /*
949 * We don't need to issue any invalidation here, as we'll invalidate
950 * the STE when installing the new entry anyway.
951 */
952 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
953#ifdef __BIG_ENDIAN
954 CTXDESC_CD_0_ENDI |
955#endif
956 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
957 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
958 CTXDESC_CD_0_V;
959 cfg->cdptr[0] = cpu_to_le64(val);
960
961 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
962 cfg->cdptr[1] = cpu_to_le64(val);
963
964 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
965}
966
967/* Stream table manipulation functions */
968static void
969arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
970{
971 u64 val = 0;
972
973 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
974 << STRTAB_L1_DESC_SPAN_SHIFT;
975 val |= desc->l2ptr_dma &
976 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
977
978 *dst = cpu_to_le64(val);
979}
980
981static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
982{
983 struct arm_smmu_cmdq_ent cmd = {
984 .opcode = CMDQ_OP_CFGI_STE,
985 .cfgi = {
986 .sid = sid,
987 .leaf = true,
988 },
989 };
990
991 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
992 cmd.opcode = CMDQ_OP_CMD_SYNC;
993 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
994}
995
996static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
997 __le64 *dst, struct arm_smmu_strtab_ent *ste)
998{
999 /*
1000 * This is hideously complicated, but we only really care about
1001 * three cases at the moment:
1002 *
1003 * 1. Invalid (all zero) -> bypass (init)
1004 * 2. Bypass -> translation (attach)
1005 * 3. Translation -> bypass (detach)
1006 *
1007 * Given that we can't update the STE atomically and the SMMU
1008 * doesn't read the thing in a defined order, that leaves us
1009 * with the following maintenance requirements:
1010 *
1011 * 1. Update Config, return (init time STEs aren't live)
1012 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1013 * 3. Update Config, sync
1014 */
1015 u64 val = le64_to_cpu(dst[0]);
1016 bool ste_live = false;
1017 struct arm_smmu_cmdq_ent prefetch_cmd = {
1018 .opcode = CMDQ_OP_PREFETCH_CFG,
1019 .prefetch = {
1020 .sid = sid,
1021 },
1022 };
1023
1024 if (val & STRTAB_STE_0_V) {
1025 u64 cfg;
1026
1027 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1028 switch (cfg) {
1029 case STRTAB_STE_0_CFG_BYPASS:
1030 break;
1031 case STRTAB_STE_0_CFG_S1_TRANS:
1032 case STRTAB_STE_0_CFG_S2_TRANS:
1033 ste_live = true;
1034 break;
1035 default:
1036 BUG(); /* STE corruption */
1037 }
1038 }
1039
1040 /* Nuke the existing Config, as we're going to rewrite it */
1041 val &= ~(STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT);
1042
1043 if (ste->valid)
1044 val |= STRTAB_STE_0_V;
1045 else
1046 val &= ~STRTAB_STE_0_V;
1047
1048 if (ste->bypass) {
1049 val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
1050 : STRTAB_STE_0_CFG_BYPASS;
1051 dst[0] = cpu_to_le64(val);
Will Deacona0eacd82015-11-18 18:15:51 +00001052 dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
1053 << STRTAB_STE_1_SHCFG_SHIFT);
Will Deacon48ec83b2015-05-27 17:25:59 +01001054 dst[2] = 0; /* Nuke the VMID */
1055 if (ste_live)
1056 arm_smmu_sync_ste_for_sid(smmu, sid);
1057 return;
1058 }
1059
1060 if (ste->s1_cfg) {
1061 BUG_ON(ste_live);
1062 dst[1] = cpu_to_le64(
1063 STRTAB_STE_1_S1C_CACHE_WBRA
1064 << STRTAB_STE_1_S1CIR_SHIFT |
1065 STRTAB_STE_1_S1C_CACHE_WBRA
1066 << STRTAB_STE_1_S1COR_SHIFT |
1067 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
Will Deacon48ec83b2015-05-27 17:25:59 +01001068#ifdef CONFIG_PCI_ATS
1069 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1070#endif
1071 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1072
Prem Mallappa6380be02015-12-14 22:01:23 +05301073 if (smmu->features & ARM_SMMU_FEAT_STALLS)
1074 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1075
Will Deacon48ec83b2015-05-27 17:25:59 +01001076 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1077 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1078 STRTAB_STE_0_CFG_S1_TRANS;
1079
1080 }
1081
1082 if (ste->s2_cfg) {
1083 BUG_ON(ste_live);
1084 dst[2] = cpu_to_le64(
1085 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1086 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1087 << STRTAB_STE_2_VTCR_SHIFT |
1088#ifdef __BIG_ENDIAN
1089 STRTAB_STE_2_S2ENDI |
1090#endif
1091 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1092 STRTAB_STE_2_S2R);
1093
1094 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1095 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1096
1097 val |= STRTAB_STE_0_CFG_S2_TRANS;
1098 }
1099
1100 arm_smmu_sync_ste_for_sid(smmu, sid);
1101 dst[0] = cpu_to_le64(val);
1102 arm_smmu_sync_ste_for_sid(smmu, sid);
1103
1104 /* It's likely that we'll want to use the new STE soon */
Zhen Lei5e929462015-07-07 04:30:18 +01001105 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1106 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
Will Deacon48ec83b2015-05-27 17:25:59 +01001107}
1108
1109static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1110{
1111 unsigned int i;
1112 struct arm_smmu_strtab_ent ste = {
1113 .valid = true,
1114 .bypass = true,
1115 };
1116
1117 for (i = 0; i < nent; ++i) {
1118 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1119 strtab += STRTAB_STE_DWORDS;
1120 }
1121}
1122
1123static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1124{
1125 size_t size;
1126 void *strtab;
1127 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1128 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1129
1130 if (desc->l2ptr)
1131 return 0;
1132
1133 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
Zhen Lei69146e72015-06-26 09:32:58 +01001134 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
Will Deacon48ec83b2015-05-27 17:25:59 +01001135
1136 desc->span = STRTAB_SPLIT + 1;
Will Deacon04fa26c2015-10-30 18:12:41 +00001137 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1138 GFP_KERNEL | __GFP_ZERO);
Will Deacon48ec83b2015-05-27 17:25:59 +01001139 if (!desc->l2ptr) {
1140 dev_err(smmu->dev,
1141 "failed to allocate l2 stream table for SID %u\n",
1142 sid);
1143 return -ENOMEM;
1144 }
1145
1146 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1147 arm_smmu_write_strtab_l1_desc(strtab, desc);
1148 return 0;
1149}
1150
1151/* IRQ and event handlers */
1152static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1153{
1154 int i;
1155 struct arm_smmu_device *smmu = dev;
1156 struct arm_smmu_queue *q = &smmu->evtq.q;
1157 u64 evt[EVTQ_ENT_DWORDS];
1158
1159 while (!queue_remove_raw(q, evt)) {
1160 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1161
1162 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1163 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1164 dev_info(smmu->dev, "\t0x%016llx\n",
1165 (unsigned long long)evt[i]);
1166 }
1167
1168 /* Sync our overflow flag, as we believe we're up to speed */
1169 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1170 return IRQ_HANDLED;
1171}
1172
1173static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev)
1174{
1175 irqreturn_t ret = IRQ_WAKE_THREAD;
1176 struct arm_smmu_device *smmu = dev;
1177 struct arm_smmu_queue *q = &smmu->evtq.q;
1178
1179 /*
1180 * Not much we can do on overflow, so scream and pretend we're
1181 * trying harder.
1182 */
1183 if (queue_sync_prod(q) == -EOVERFLOW)
1184 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1185 else if (queue_empty(q))
1186 ret = IRQ_NONE;
1187
1188 return ret;
1189}
1190
1191static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1192{
1193 struct arm_smmu_device *smmu = dev;
1194 struct arm_smmu_queue *q = &smmu->priq.q;
1195 u64 evt[PRIQ_ENT_DWORDS];
1196
1197 while (!queue_remove_raw(q, evt)) {
1198 u32 sid, ssid;
1199 u16 grpid;
1200 bool ssv, last;
1201
1202 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1203 ssv = evt[0] & PRIQ_0_SSID_V;
1204 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1205 last = evt[0] & PRIQ_0_PRG_LAST;
1206 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1207
1208 dev_info(smmu->dev, "unexpected PRI request received:\n");
1209 dev_info(smmu->dev,
1210 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1211 sid, ssid, grpid, last ? "L" : "",
1212 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1213 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1214 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1215 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1216 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1217
1218 if (last) {
1219 struct arm_smmu_cmdq_ent cmd = {
1220 .opcode = CMDQ_OP_PRI_RESP,
1221 .substream_valid = ssv,
1222 .pri = {
1223 .sid = sid,
1224 .ssid = ssid,
1225 .grpid = grpid,
1226 .resp = PRI_RESP_DENY,
1227 },
1228 };
1229
1230 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1231 }
1232 }
1233
1234 /* Sync our overflow flag, as we believe we're up to speed */
1235 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1236 return IRQ_HANDLED;
1237}
1238
1239static irqreturn_t arm_smmu_priq_handler(int irq, void *dev)
1240{
1241 irqreturn_t ret = IRQ_WAKE_THREAD;
1242 struct arm_smmu_device *smmu = dev;
1243 struct arm_smmu_queue *q = &smmu->priq.q;
1244
1245 /* PRIQ overflow indicates a programming error */
1246 if (queue_sync_prod(q) == -EOVERFLOW)
1247 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1248 else if (queue_empty(q))
1249 ret = IRQ_NONE;
1250
1251 return ret;
1252}
1253
1254static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1255{
1256 /* We don't actually use CMD_SYNC interrupts for anything */
1257 return IRQ_HANDLED;
1258}
1259
1260static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1261
1262static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1263{
Prem Mallappa324ba102015-12-14 22:01:14 +05301264 u32 gerror, gerrorn, active;
Will Deacon48ec83b2015-05-27 17:25:59 +01001265 struct arm_smmu_device *smmu = dev;
1266
1267 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1268 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1269
Prem Mallappa324ba102015-12-14 22:01:14 +05301270 active = gerror ^ gerrorn;
1271 if (!(active & GERROR_ERR_MASK))
Will Deacon48ec83b2015-05-27 17:25:59 +01001272 return IRQ_NONE; /* No errors pending */
1273
1274 dev_warn(smmu->dev,
1275 "unexpected global error reported (0x%08x), this could be serious\n",
Prem Mallappa324ba102015-12-14 22:01:14 +05301276 active);
Will Deacon48ec83b2015-05-27 17:25:59 +01001277
Prem Mallappa324ba102015-12-14 22:01:14 +05301278 if (active & GERROR_SFM_ERR) {
Will Deacon48ec83b2015-05-27 17:25:59 +01001279 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1280 arm_smmu_device_disable(smmu);
1281 }
1282
Prem Mallappa324ba102015-12-14 22:01:14 +05301283 if (active & GERROR_MSI_GERROR_ABT_ERR)
Will Deacon48ec83b2015-05-27 17:25:59 +01001284 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1285
Prem Mallappa324ba102015-12-14 22:01:14 +05301286 if (active & GERROR_MSI_PRIQ_ABT_ERR) {
Will Deacon48ec83b2015-05-27 17:25:59 +01001287 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1288 arm_smmu_priq_handler(irq, smmu->dev);
1289 }
1290
Prem Mallappa324ba102015-12-14 22:01:14 +05301291 if (active & GERROR_MSI_EVTQ_ABT_ERR) {
Will Deacon48ec83b2015-05-27 17:25:59 +01001292 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1293 arm_smmu_evtq_handler(irq, smmu->dev);
1294 }
1295
Prem Mallappa324ba102015-12-14 22:01:14 +05301296 if (active & GERROR_MSI_CMDQ_ABT_ERR) {
Will Deacon48ec83b2015-05-27 17:25:59 +01001297 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1298 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1299 }
1300
Prem Mallappa324ba102015-12-14 22:01:14 +05301301 if (active & GERROR_PRIQ_ABT_ERR)
Will Deacon48ec83b2015-05-27 17:25:59 +01001302 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1303
Prem Mallappa324ba102015-12-14 22:01:14 +05301304 if (active & GERROR_EVTQ_ABT_ERR)
Will Deacon48ec83b2015-05-27 17:25:59 +01001305 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1306
Prem Mallappa324ba102015-12-14 22:01:14 +05301307 if (active & GERROR_CMDQ_ERR)
Will Deacon48ec83b2015-05-27 17:25:59 +01001308 arm_smmu_cmdq_skip_err(smmu);
1309
1310 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1311 return IRQ_HANDLED;
1312}
1313
1314/* IO_PGTABLE API */
1315static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1316{
1317 struct arm_smmu_cmdq_ent cmd;
1318
1319 cmd.opcode = CMDQ_OP_CMD_SYNC;
1320 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1321}
1322
1323static void arm_smmu_tlb_sync(void *cookie)
1324{
1325 struct arm_smmu_domain *smmu_domain = cookie;
1326 __arm_smmu_tlb_sync(smmu_domain->smmu);
1327}
1328
1329static void arm_smmu_tlb_inv_context(void *cookie)
1330{
1331 struct arm_smmu_domain *smmu_domain = cookie;
1332 struct arm_smmu_device *smmu = smmu_domain->smmu;
1333 struct arm_smmu_cmdq_ent cmd;
1334
1335 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1336 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1337 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1338 cmd.tlbi.vmid = 0;
1339 } else {
1340 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1341 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1342 }
1343
1344 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1345 __arm_smmu_tlb_sync(smmu);
1346}
1347
1348static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
Robin Murphy06c610e2015-12-07 18:18:53 +00001349 size_t granule, bool leaf, void *cookie)
Will Deacon48ec83b2015-05-27 17:25:59 +01001350{
1351 struct arm_smmu_domain *smmu_domain = cookie;
1352 struct arm_smmu_device *smmu = smmu_domain->smmu;
1353 struct arm_smmu_cmdq_ent cmd = {
1354 .tlbi = {
1355 .leaf = leaf,
1356 .addr = iova,
1357 },
1358 };
1359
1360 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1361 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1362 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1363 } else {
1364 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1365 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1366 }
1367
Robin Murphy75df1382015-12-07 18:18:52 +00001368 do {
1369 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1370 cmd.tlbi.addr += granule;
1371 } while (size -= granule);
Will Deacon48ec83b2015-05-27 17:25:59 +01001372}
1373
Will Deacon48ec83b2015-05-27 17:25:59 +01001374static struct iommu_gather_ops arm_smmu_gather_ops = {
1375 .tlb_flush_all = arm_smmu_tlb_inv_context,
1376 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1377 .tlb_sync = arm_smmu_tlb_sync,
Will Deacon48ec83b2015-05-27 17:25:59 +01001378};
1379
1380/* IOMMU API */
1381static bool arm_smmu_capable(enum iommu_cap cap)
1382{
1383 switch (cap) {
1384 case IOMMU_CAP_CACHE_COHERENCY:
1385 return true;
1386 case IOMMU_CAP_INTR_REMAP:
1387 return true; /* MSIs are just memory writes */
1388 case IOMMU_CAP_NOEXEC:
1389 return true;
1390 default:
1391 return false;
1392 }
1393}
1394
1395static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1396{
1397 struct arm_smmu_domain *smmu_domain;
1398
1399 if (type != IOMMU_DOMAIN_UNMANAGED)
1400 return NULL;
1401
1402 /*
1403 * Allocate the domain and initialise some of its data structures.
1404 * We can't really do anything meaningful until we've added a
1405 * master.
1406 */
1407 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1408 if (!smmu_domain)
1409 return NULL;
1410
1411 mutex_init(&smmu_domain->init_mutex);
1412 spin_lock_init(&smmu_domain->pgtbl_lock);
1413 return &smmu_domain->domain;
1414}
1415
1416static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1417{
1418 int idx, size = 1 << span;
1419
1420 do {
1421 idx = find_first_zero_bit(map, size);
1422 if (idx == size)
1423 return -ENOSPC;
1424 } while (test_and_set_bit(idx, map));
1425
1426 return idx;
1427}
1428
1429static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1430{
1431 clear_bit(idx, map);
1432}
1433
1434static void arm_smmu_domain_free(struct iommu_domain *domain)
1435{
1436 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1437 struct arm_smmu_device *smmu = smmu_domain->smmu;
1438
Markus Elfringa6e08fb2015-06-29 17:47:43 +01001439 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
Will Deacon48ec83b2015-05-27 17:25:59 +01001440
1441 /* Free the CD and ASID, if we allocated them */
1442 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1443 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1444
1445 if (cfg->cdptr) {
Will Deacon04fa26c2015-10-30 18:12:41 +00001446 dmam_free_coherent(smmu_domain->smmu->dev,
1447 CTXDESC_CD_DWORDS << 3,
1448 cfg->cdptr,
1449 cfg->cdptr_dma);
Will Deacon48ec83b2015-05-27 17:25:59 +01001450
1451 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1452 }
1453 } else {
1454 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1455 if (cfg->vmid)
1456 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1457 }
1458
1459 kfree(smmu_domain);
1460}
1461
1462static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1463 struct io_pgtable_cfg *pgtbl_cfg)
1464{
1465 int ret;
Will Deaconc0733a22015-10-13 17:51:14 +01001466 int asid;
Will Deacon48ec83b2015-05-27 17:25:59 +01001467 struct arm_smmu_device *smmu = smmu_domain->smmu;
1468 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1469
1470 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1471 if (IS_ERR_VALUE(asid))
1472 return asid;
1473
Will Deacon04fa26c2015-10-30 18:12:41 +00001474 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1475 &cfg->cdptr_dma,
1476 GFP_KERNEL | __GFP_ZERO);
Will Deacon48ec83b2015-05-27 17:25:59 +01001477 if (!cfg->cdptr) {
1478 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
Will Deaconc0733a22015-10-13 17:51:14 +01001479 ret = -ENOMEM;
Will Deacon48ec83b2015-05-27 17:25:59 +01001480 goto out_free_asid;
1481 }
1482
Will Deaconc0733a22015-10-13 17:51:14 +01001483 cfg->cd.asid = (u16)asid;
Will Deacon48ec83b2015-05-27 17:25:59 +01001484 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1485 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1486 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1487 return 0;
1488
1489out_free_asid:
1490 arm_smmu_bitmap_free(smmu->asid_map, asid);
1491 return ret;
1492}
1493
1494static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1495 struct io_pgtable_cfg *pgtbl_cfg)
1496{
Will Deaconc0733a22015-10-13 17:51:14 +01001497 int vmid;
Will Deacon48ec83b2015-05-27 17:25:59 +01001498 struct arm_smmu_device *smmu = smmu_domain->smmu;
1499 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1500
1501 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1502 if (IS_ERR_VALUE(vmid))
1503 return vmid;
1504
Will Deaconc0733a22015-10-13 17:51:14 +01001505 cfg->vmid = (u16)vmid;
Will Deacon48ec83b2015-05-27 17:25:59 +01001506 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1507 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1508 return 0;
1509}
1510
1511static struct iommu_ops arm_smmu_ops;
1512
1513static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1514{
1515 int ret;
1516 unsigned long ias, oas;
1517 enum io_pgtable_fmt fmt;
1518 struct io_pgtable_cfg pgtbl_cfg;
1519 struct io_pgtable_ops *pgtbl_ops;
1520 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1521 struct io_pgtable_cfg *);
1522 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1523 struct arm_smmu_device *smmu = smmu_domain->smmu;
1524
1525 /* Restrict the stage to what we can actually support */
1526 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1527 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1528 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1529 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1530
1531 switch (smmu_domain->stage) {
1532 case ARM_SMMU_DOMAIN_S1:
1533 ias = VA_BITS;
1534 oas = smmu->ias;
1535 fmt = ARM_64_LPAE_S1;
1536 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1537 break;
1538 case ARM_SMMU_DOMAIN_NESTED:
1539 case ARM_SMMU_DOMAIN_S2:
1540 ias = smmu->ias;
1541 oas = smmu->oas;
1542 fmt = ARM_64_LPAE_S2;
1543 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1544 break;
1545 default:
1546 return -EINVAL;
1547 }
1548
1549 pgtbl_cfg = (struct io_pgtable_cfg) {
1550 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
1551 .ias = ias,
1552 .oas = oas,
1553 .tlb = &arm_smmu_gather_ops,
Robin Murphybdc6d972015-07-29 19:46:07 +01001554 .iommu_dev = smmu->dev,
Will Deacon48ec83b2015-05-27 17:25:59 +01001555 };
1556
1557 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1558 if (!pgtbl_ops)
1559 return -ENOMEM;
1560
1561 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1562 smmu_domain->pgtbl_ops = pgtbl_ops;
1563
1564 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1565 if (IS_ERR_VALUE(ret))
1566 free_io_pgtable_ops(pgtbl_ops);
1567
1568 return ret;
1569}
1570
1571static struct arm_smmu_group *arm_smmu_group_get(struct device *dev)
1572{
1573 struct iommu_group *group;
1574 struct arm_smmu_group *smmu_group;
1575
1576 group = iommu_group_get(dev);
1577 if (!group)
1578 return NULL;
1579
1580 smmu_group = iommu_group_get_iommudata(group);
1581 iommu_group_put(group);
1582 return smmu_group;
1583}
1584
1585static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1586{
1587 __le64 *step;
1588 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1589
1590 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1591 struct arm_smmu_strtab_l1_desc *l1_desc;
1592 int idx;
1593
1594 /* Two-level walk */
1595 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1596 l1_desc = &cfg->l1_desc[idx];
1597 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1598 step = &l1_desc->l2ptr[idx];
1599 } else {
1600 /* Simple linear lookup */
1601 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1602 }
1603
1604 return step;
1605}
1606
1607static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
1608{
1609 int i;
1610 struct arm_smmu_domain *smmu_domain = smmu_group->domain;
1611 struct arm_smmu_strtab_ent *ste = &smmu_group->ste;
1612 struct arm_smmu_device *smmu = smmu_group->smmu;
1613
1614 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1615 ste->s1_cfg = &smmu_domain->s1_cfg;
1616 ste->s2_cfg = NULL;
1617 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1618 } else {
1619 ste->s1_cfg = NULL;
1620 ste->s2_cfg = &smmu_domain->s2_cfg;
1621 }
1622
1623 for (i = 0; i < smmu_group->num_sids; ++i) {
1624 u32 sid = smmu_group->sids[i];
1625 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1626
1627 arm_smmu_write_strtab_ent(smmu, sid, step, ste);
1628 }
1629
1630 return 0;
1631}
1632
1633static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1634{
1635 int ret = 0;
1636 struct arm_smmu_device *smmu;
1637 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1638 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1639
1640 if (!smmu_group)
1641 return -ENOENT;
1642
1643 /* Already attached to a different domain? */
1644 if (smmu_group->domain && smmu_group->domain != smmu_domain)
1645 return -EEXIST;
1646
1647 smmu = smmu_group->smmu;
1648 mutex_lock(&smmu_domain->init_mutex);
1649
1650 if (!smmu_domain->smmu) {
1651 smmu_domain->smmu = smmu;
1652 ret = arm_smmu_domain_finalise(domain);
1653 if (ret) {
1654 smmu_domain->smmu = NULL;
1655 goto out_unlock;
1656 }
1657 } else if (smmu_domain->smmu != smmu) {
1658 dev_err(dev,
1659 "cannot attach to SMMU %s (upstream of %s)\n",
1660 dev_name(smmu_domain->smmu->dev),
1661 dev_name(smmu->dev));
1662 ret = -ENXIO;
1663 goto out_unlock;
1664 }
1665
1666 /* Group already attached to this domain? */
1667 if (smmu_group->domain)
1668 goto out_unlock;
1669
1670 smmu_group->domain = smmu_domain;
1671 smmu_group->ste.bypass = false;
1672
1673 ret = arm_smmu_install_ste_for_group(smmu_group);
1674 if (IS_ERR_VALUE(ret))
1675 smmu_group->domain = NULL;
1676
1677out_unlock:
1678 mutex_unlock(&smmu_domain->init_mutex);
1679 return ret;
1680}
1681
1682static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
1683{
1684 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1685 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1686
1687 BUG_ON(!smmu_domain);
1688 BUG_ON(!smmu_group);
1689
1690 mutex_lock(&smmu_domain->init_mutex);
1691 BUG_ON(smmu_group->domain != smmu_domain);
1692
1693 smmu_group->ste.bypass = true;
1694 if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group)))
1695 dev_warn(dev, "failed to install bypass STE\n");
1696
1697 smmu_group->domain = NULL;
1698 mutex_unlock(&smmu_domain->init_mutex);
1699}
1700
1701static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1702 phys_addr_t paddr, size_t size, int prot)
1703{
1704 int ret;
1705 unsigned long flags;
1706 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1707 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1708
1709 if (!ops)
1710 return -ENODEV;
1711
1712 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1713 ret = ops->map(ops, iova, paddr, size, prot);
1714 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1715 return ret;
1716}
1717
1718static size_t
1719arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1720{
1721 size_t ret;
1722 unsigned long flags;
1723 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1724 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1725
1726 if (!ops)
1727 return 0;
1728
1729 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1730 ret = ops->unmap(ops, iova, size);
1731 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1732 return ret;
1733}
1734
1735static phys_addr_t
1736arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1737{
1738 phys_addr_t ret;
1739 unsigned long flags;
1740 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1741 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1742
1743 if (!ops)
1744 return 0;
1745
1746 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1747 ret = ops->iova_to_phys(ops, iova);
1748 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1749
1750 return ret;
1751}
1752
1753static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp)
1754{
1755 *(u32 *)sidp = alias;
1756 return 0; /* Continue walking */
1757}
1758
1759static void __arm_smmu_release_pci_iommudata(void *data)
1760{
1761 kfree(data);
1762}
1763
1764static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev)
1765{
1766 struct device_node *of_node;
Will Deacon941a8022015-08-11 16:25:10 +01001767 struct platform_device *smmu_pdev;
1768 struct arm_smmu_device *smmu = NULL;
Will Deacon48ec83b2015-05-27 17:25:59 +01001769 struct pci_bus *bus = pdev->bus;
1770
1771 /* Walk up to the root bus */
1772 while (!pci_is_root_bus(bus))
1773 bus = bus->parent;
1774
1775 /* Follow the "iommus" phandle from the host controller */
1776 of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0);
1777 if (!of_node)
1778 return NULL;
1779
1780 /* See if we can find an SMMU corresponding to the phandle */
Will Deacon941a8022015-08-11 16:25:10 +01001781 smmu_pdev = of_find_device_by_node(of_node);
1782 if (smmu_pdev)
1783 smmu = platform_get_drvdata(smmu_pdev);
1784
Will Deacon48ec83b2015-05-27 17:25:59 +01001785 of_node_put(of_node);
1786 return smmu;
1787}
1788
1789static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1790{
1791 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1792
1793 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1794 limit *= 1UL << STRTAB_SPLIT;
1795
1796 return sid < limit;
1797}
1798
1799static int arm_smmu_add_device(struct device *dev)
1800{
1801 int i, ret;
1802 u32 sid, *sids;
1803 struct pci_dev *pdev;
1804 struct iommu_group *group;
1805 struct arm_smmu_group *smmu_group;
1806 struct arm_smmu_device *smmu;
1807
1808 /* We only support PCI, for now */
1809 if (!dev_is_pci(dev))
1810 return -ENODEV;
1811
1812 pdev = to_pci_dev(dev);
1813 group = iommu_group_get_for_dev(dev);
1814 if (IS_ERR(group))
1815 return PTR_ERR(group);
1816
1817 smmu_group = iommu_group_get_iommudata(group);
1818 if (!smmu_group) {
1819 smmu = arm_smmu_get_for_pci_dev(pdev);
1820 if (!smmu) {
1821 ret = -ENOENT;
Peng Fan9a4a9d82015-11-20 16:56:18 +08001822 goto out_remove_dev;
Will Deacon48ec83b2015-05-27 17:25:59 +01001823 }
1824
1825 smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
1826 if (!smmu_group) {
1827 ret = -ENOMEM;
Peng Fan9a4a9d82015-11-20 16:56:18 +08001828 goto out_remove_dev;
Will Deacon48ec83b2015-05-27 17:25:59 +01001829 }
1830
1831 smmu_group->ste.valid = true;
1832 smmu_group->smmu = smmu;
1833 iommu_group_set_iommudata(group, smmu_group,
1834 __arm_smmu_release_pci_iommudata);
1835 } else {
1836 smmu = smmu_group->smmu;
1837 }
1838
1839 /* Assume SID == RID until firmware tells us otherwise */
1840 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1841 for (i = 0; i < smmu_group->num_sids; ++i) {
1842 /* If we already know about this SID, then we're done */
1843 if (smmu_group->sids[i] == sid)
Peng Fan9a4a9d82015-11-20 16:56:18 +08001844 goto out_put_group;
Will Deacon48ec83b2015-05-27 17:25:59 +01001845 }
1846
1847 /* Check the SID is in range of the SMMU and our stream table */
1848 if (!arm_smmu_sid_in_range(smmu, sid)) {
1849 ret = -ERANGE;
Peng Fan9a4a9d82015-11-20 16:56:18 +08001850 goto out_remove_dev;
Will Deacon48ec83b2015-05-27 17:25:59 +01001851 }
1852
1853 /* Ensure l2 strtab is initialised */
1854 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1855 ret = arm_smmu_init_l2_strtab(smmu, sid);
1856 if (ret)
Peng Fan9a4a9d82015-11-20 16:56:18 +08001857 goto out_remove_dev;
Will Deacon48ec83b2015-05-27 17:25:59 +01001858 }
1859
1860 /* Resize the SID array for the group */
1861 smmu_group->num_sids++;
1862 sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids),
1863 GFP_KERNEL);
1864 if (!sids) {
1865 smmu_group->num_sids--;
1866 ret = -ENOMEM;
Peng Fan9a4a9d82015-11-20 16:56:18 +08001867 goto out_remove_dev;
Will Deacon48ec83b2015-05-27 17:25:59 +01001868 }
1869
1870 /* Add the new SID */
1871 sids[smmu_group->num_sids - 1] = sid;
1872 smmu_group->sids = sids;
Will Deacon48ec83b2015-05-27 17:25:59 +01001873
1874out_put_group:
1875 iommu_group_put(group);
Peng Fan9a4a9d82015-11-20 16:56:18 +08001876 return 0;
1877
1878out_remove_dev:
1879 iommu_group_remove_device(dev);
1880 iommu_group_put(group);
Will Deacon48ec83b2015-05-27 17:25:59 +01001881 return ret;
1882}
1883
1884static void arm_smmu_remove_device(struct device *dev)
1885{
1886 iommu_group_remove_device(dev);
1887}
1888
1889static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1890 enum iommu_attr attr, void *data)
1891{
1892 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1893
1894 switch (attr) {
1895 case DOMAIN_ATTR_NESTING:
1896 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1897 return 0;
1898 default:
1899 return -ENODEV;
1900 }
1901}
1902
1903static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1904 enum iommu_attr attr, void *data)
1905{
1906 int ret = 0;
1907 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1908
1909 mutex_lock(&smmu_domain->init_mutex);
1910
1911 switch (attr) {
1912 case DOMAIN_ATTR_NESTING:
1913 if (smmu_domain->smmu) {
1914 ret = -EPERM;
1915 goto out_unlock;
1916 }
1917
1918 if (*(int *)data)
1919 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1920 else
1921 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1922
1923 break;
1924 default:
1925 ret = -ENODEV;
1926 }
1927
1928out_unlock:
1929 mutex_unlock(&smmu_domain->init_mutex);
1930 return ret;
1931}
1932
1933static struct iommu_ops arm_smmu_ops = {
1934 .capable = arm_smmu_capable,
1935 .domain_alloc = arm_smmu_domain_alloc,
1936 .domain_free = arm_smmu_domain_free,
1937 .attach_dev = arm_smmu_attach_dev,
1938 .detach_dev = arm_smmu_detach_dev,
1939 .map = arm_smmu_map,
1940 .unmap = arm_smmu_unmap,
1941 .iova_to_phys = arm_smmu_iova_to_phys,
1942 .add_device = arm_smmu_add_device,
1943 .remove_device = arm_smmu_remove_device,
Joerg Roedelaf659932015-10-21 23:51:41 +02001944 .device_group = pci_device_group,
Will Deacon48ec83b2015-05-27 17:25:59 +01001945 .domain_get_attr = arm_smmu_domain_get_attr,
1946 .domain_set_attr = arm_smmu_domain_set_attr,
1947 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1948};
1949
1950/* Probing and initialisation functions */
1951static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1952 struct arm_smmu_queue *q,
1953 unsigned long prod_off,
1954 unsigned long cons_off,
1955 size_t dwords)
1956{
1957 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1958
Will Deacon04fa26c2015-10-30 18:12:41 +00001959 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
Will Deacon48ec83b2015-05-27 17:25:59 +01001960 if (!q->base) {
1961 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1962 qsz);
1963 return -ENOMEM;
1964 }
1965
1966 q->prod_reg = smmu->base + prod_off;
1967 q->cons_reg = smmu->base + cons_off;
1968 q->ent_dwords = dwords;
1969
1970 q->q_base = Q_BASE_RWA;
1971 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1972 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1973 << Q_BASE_LOG2SIZE_SHIFT;
1974
1975 q->prod = q->cons = 0;
1976 return 0;
1977}
1978
Will Deacon48ec83b2015-05-27 17:25:59 +01001979static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1980{
1981 int ret;
1982
1983 /* cmdq */
1984 spin_lock_init(&smmu->cmdq.lock);
1985 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1986 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1987 if (ret)
Will Deacon04fa26c2015-10-30 18:12:41 +00001988 return ret;
Will Deacon48ec83b2015-05-27 17:25:59 +01001989
1990 /* evtq */
1991 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1992 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
1993 if (ret)
Will Deacon04fa26c2015-10-30 18:12:41 +00001994 return ret;
Will Deacon48ec83b2015-05-27 17:25:59 +01001995
1996 /* priq */
1997 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
1998 return 0;
1999
Will Deacon04fa26c2015-10-30 18:12:41 +00002000 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2001 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
Will Deacon48ec83b2015-05-27 17:25:59 +01002002}
2003
2004static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2005{
2006 unsigned int i;
2007 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2008 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2009 void *strtab = smmu->strtab_cfg.strtab;
2010
2011 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2012 if (!cfg->l1_desc) {
2013 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2014 return -ENOMEM;
2015 }
2016
2017 for (i = 0; i < cfg->num_l1_ents; ++i) {
2018 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2019 strtab += STRTAB_L1_DESC_DWORDS << 3;
2020 }
2021
2022 return 0;
2023}
2024
2025static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2026{
2027 void *strtab;
2028 u64 reg;
Will Deacond2e88e72015-06-30 10:02:28 +01002029 u32 size, l1size;
Will Deacon48ec83b2015-05-27 17:25:59 +01002030 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2031
Will Deacon28c8b402015-07-16 17:50:12 +01002032 /*
2033 * If we can resolve everything with a single L2 table, then we
2034 * just need a single L1 descriptor. Otherwise, calculate the L1
2035 * size, capped to the SIDSIZE.
2036 */
2037 if (smmu->sid_bits < STRTAB_SPLIT) {
2038 size = 0;
2039 } else {
2040 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2041 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2042 }
Will Deacond2e88e72015-06-30 10:02:28 +01002043 cfg->num_l1_ents = 1 << size;
2044
2045 size += STRTAB_SPLIT;
2046 if (size < smmu->sid_bits)
Will Deacon48ec83b2015-05-27 17:25:59 +01002047 dev_warn(smmu->dev,
2048 "2-level strtab only covers %u/%u bits of SID\n",
Will Deacond2e88e72015-06-30 10:02:28 +01002049 size, smmu->sid_bits);
Will Deacon48ec83b2015-05-27 17:25:59 +01002050
Will Deacond2e88e72015-06-30 10:02:28 +01002051 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
Will Deacon04fa26c2015-10-30 18:12:41 +00002052 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2053 GFP_KERNEL | __GFP_ZERO);
Will Deacon48ec83b2015-05-27 17:25:59 +01002054 if (!strtab) {
2055 dev_err(smmu->dev,
2056 "failed to allocate l1 stream table (%u bytes)\n",
2057 size);
2058 return -ENOMEM;
2059 }
2060 cfg->strtab = strtab;
2061
2062 /* Configure strtab_base_cfg for 2 levels */
2063 reg = STRTAB_BASE_CFG_FMT_2LVL;
2064 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2065 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2066 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2067 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2068 cfg->strtab_base_cfg = reg;
2069
Will Deacon04fa26c2015-10-30 18:12:41 +00002070 return arm_smmu_init_l1_strtab(smmu);
Will Deacon48ec83b2015-05-27 17:25:59 +01002071}
2072
2073static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2074{
2075 void *strtab;
2076 u64 reg;
2077 u32 size;
2078 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2079
2080 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
Will Deacon04fa26c2015-10-30 18:12:41 +00002081 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2082 GFP_KERNEL | __GFP_ZERO);
Will Deacon48ec83b2015-05-27 17:25:59 +01002083 if (!strtab) {
2084 dev_err(smmu->dev,
2085 "failed to allocate linear stream table (%u bytes)\n",
2086 size);
2087 return -ENOMEM;
2088 }
2089 cfg->strtab = strtab;
2090 cfg->num_l1_ents = 1 << smmu->sid_bits;
2091
2092 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2093 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2094 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2095 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2096 cfg->strtab_base_cfg = reg;
2097
2098 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2099 return 0;
2100}
2101
2102static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2103{
2104 u64 reg;
2105 int ret;
2106
2107 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2108 ret = arm_smmu_init_strtab_2lvl(smmu);
2109 else
2110 ret = arm_smmu_init_strtab_linear(smmu);
2111
2112 if (ret)
2113 return ret;
2114
2115 /* Set the strtab base address */
2116 reg = smmu->strtab_cfg.strtab_dma &
2117 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2118 reg |= STRTAB_BASE_RA;
2119 smmu->strtab_cfg.strtab_base = reg;
2120
2121 /* Allocate the first VMID for stage-2 bypass STEs */
2122 set_bit(0, smmu->vmid_map);
2123 return 0;
2124}
2125
Will Deacon48ec83b2015-05-27 17:25:59 +01002126static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2127{
2128 int ret;
2129
2130 ret = arm_smmu_init_queues(smmu);
2131 if (ret)
2132 return ret;
2133
Will Deacon04fa26c2015-10-30 18:12:41 +00002134 return arm_smmu_init_strtab(smmu);
Will Deacon48ec83b2015-05-27 17:25:59 +01002135}
2136
2137static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2138 unsigned int reg_off, unsigned int ack_off)
2139{
2140 u32 reg;
2141
2142 writel_relaxed(val, smmu->base + reg_off);
2143 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2144 1, ARM_SMMU_POLL_TIMEOUT_US);
2145}
2146
Marc Zyngier166bdbd2015-10-13 18:32:30 +01002147static void arm_smmu_free_msis(void *data)
2148{
2149 struct device *dev = data;
2150 platform_msi_domain_free_irqs(dev);
2151}
2152
2153static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2154{
2155 phys_addr_t doorbell;
2156 struct device *dev = msi_desc_to_dev(desc);
2157 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2158 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2159
2160 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2161 doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT;
2162
2163 writeq_relaxed(doorbell, smmu->base + cfg[0]);
2164 writel_relaxed(msg->data, smmu->base + cfg[1]);
2165 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2166}
2167
2168static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2169{
2170 struct msi_desc *desc;
2171 int ret, nvec = ARM_SMMU_MAX_MSIS;
2172 struct device *dev = smmu->dev;
2173
2174 /* Clear the MSI address regs */
2175 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2176 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2177
2178 if (smmu->features & ARM_SMMU_FEAT_PRI)
2179 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2180 else
2181 nvec--;
2182
2183 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2184 return;
2185
2186 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2187 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2188 if (ret) {
2189 dev_warn(dev, "failed to allocate MSIs\n");
2190 return;
2191 }
2192
2193 for_each_msi_entry(desc, dev) {
2194 switch (desc->platform.msi_index) {
2195 case EVTQ_MSI_INDEX:
2196 smmu->evtq.q.irq = desc->irq;
2197 break;
2198 case GERROR_MSI_INDEX:
2199 smmu->gerr_irq = desc->irq;
2200 break;
2201 case PRIQ_MSI_INDEX:
2202 smmu->priq.q.irq = desc->irq;
2203 break;
2204 default: /* Unknown */
2205 continue;
2206 }
2207 }
2208
2209 /* Add callback to free MSIs on teardown */
2210 devm_add_action(dev, arm_smmu_free_msis, dev);
2211}
2212
Will Deacon48ec83b2015-05-27 17:25:59 +01002213static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2214{
2215 int ret, irq;
Marc Zyngierccd63852015-07-15 11:55:18 +01002216 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
Will Deacon48ec83b2015-05-27 17:25:59 +01002217
2218 /* Disable IRQs first */
2219 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2220 ARM_SMMU_IRQ_CTRLACK);
2221 if (ret) {
2222 dev_err(smmu->dev, "failed to disable irqs\n");
2223 return ret;
2224 }
2225
Marc Zyngier166bdbd2015-10-13 18:32:30 +01002226 arm_smmu_setup_msis(smmu);
Will Deacon48ec83b2015-05-27 17:25:59 +01002227
Marc Zyngier166bdbd2015-10-13 18:32:30 +01002228 /* Request interrupt lines */
Will Deacon48ec83b2015-05-27 17:25:59 +01002229 irq = smmu->evtq.q.irq;
2230 if (irq) {
2231 ret = devm_request_threaded_irq(smmu->dev, irq,
2232 arm_smmu_evtq_handler,
2233 arm_smmu_evtq_thread,
2234 0, "arm-smmu-v3-evtq", smmu);
2235 if (IS_ERR_VALUE(ret))
2236 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2237 }
2238
2239 irq = smmu->cmdq.q.irq;
2240 if (irq) {
2241 ret = devm_request_irq(smmu->dev, irq,
2242 arm_smmu_cmdq_sync_handler, 0,
2243 "arm-smmu-v3-cmdq-sync", smmu);
2244 if (IS_ERR_VALUE(ret))
2245 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2246 }
2247
2248 irq = smmu->gerr_irq;
2249 if (irq) {
2250 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2251 0, "arm-smmu-v3-gerror", smmu);
2252 if (IS_ERR_VALUE(ret))
2253 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2254 }
2255
2256 if (smmu->features & ARM_SMMU_FEAT_PRI) {
Will Deacon48ec83b2015-05-27 17:25:59 +01002257 irq = smmu->priq.q.irq;
2258 if (irq) {
2259 ret = devm_request_threaded_irq(smmu->dev, irq,
2260 arm_smmu_priq_handler,
2261 arm_smmu_priq_thread,
2262 0, "arm-smmu-v3-priq",
2263 smmu);
2264 if (IS_ERR_VALUE(ret))
2265 dev_warn(smmu->dev,
2266 "failed to enable priq irq\n");
Marc Zyngierccd63852015-07-15 11:55:18 +01002267 else
2268 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
Will Deacon48ec83b2015-05-27 17:25:59 +01002269 }
2270 }
2271
2272 /* Enable interrupt generation on the SMMU */
Marc Zyngierccd63852015-07-15 11:55:18 +01002273 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
Will Deacon48ec83b2015-05-27 17:25:59 +01002274 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2275 if (ret)
2276 dev_warn(smmu->dev, "failed to enable irqs\n");
2277
2278 return 0;
2279}
2280
2281static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2282{
2283 int ret;
2284
2285 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2286 if (ret)
2287 dev_err(smmu->dev, "failed to clear cr0\n");
2288
2289 return ret;
2290}
2291
2292static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
2293{
2294 int ret;
2295 u32 reg, enables;
2296 struct arm_smmu_cmdq_ent cmd;
2297
2298 /* Clear CR0 and sync (disables SMMU and queue processing) */
2299 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2300 if (reg & CR0_SMMUEN)
2301 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2302
2303 ret = arm_smmu_device_disable(smmu);
2304 if (ret)
2305 return ret;
2306
2307 /* CR1 (table and queue memory attributes) */
2308 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2309 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2310 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2311 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2312 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2313 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2314 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2315
2316 /* CR2 (random crap) */
2317 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2318 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2319
2320 /* Stream table */
2321 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2322 smmu->base + ARM_SMMU_STRTAB_BASE);
2323 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2324 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2325
2326 /* Command queue */
2327 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2328 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2329 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2330
2331 enables = CR0_CMDQEN;
2332 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2333 ARM_SMMU_CR0ACK);
2334 if (ret) {
2335 dev_err(smmu->dev, "failed to enable command queue\n");
2336 return ret;
2337 }
2338
2339 /* Invalidate any cached configuration */
2340 cmd.opcode = CMDQ_OP_CFGI_ALL;
2341 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2342 cmd.opcode = CMDQ_OP_CMD_SYNC;
2343 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2344
2345 /* Invalidate any stale TLB entries */
2346 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2347 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2348 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2349 }
2350
2351 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2352 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2353 cmd.opcode = CMDQ_OP_CMD_SYNC;
2354 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2355
2356 /* Event queue */
2357 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2358 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2359 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2360
2361 enables |= CR0_EVTQEN;
2362 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2363 ARM_SMMU_CR0ACK);
2364 if (ret) {
2365 dev_err(smmu->dev, "failed to enable event queue\n");
2366 return ret;
2367 }
2368
2369 /* PRI queue */
2370 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2371 writeq_relaxed(smmu->priq.q.q_base,
2372 smmu->base + ARM_SMMU_PRIQ_BASE);
2373 writel_relaxed(smmu->priq.q.prod,
2374 smmu->base + ARM_SMMU_PRIQ_PROD);
2375 writel_relaxed(smmu->priq.q.cons,
2376 smmu->base + ARM_SMMU_PRIQ_CONS);
2377
2378 enables |= CR0_PRIQEN;
2379 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2380 ARM_SMMU_CR0ACK);
2381 if (ret) {
2382 dev_err(smmu->dev, "failed to enable PRI queue\n");
2383 return ret;
2384 }
2385 }
2386
2387 ret = arm_smmu_setup_irqs(smmu);
2388 if (ret) {
2389 dev_err(smmu->dev, "failed to setup irqs\n");
2390 return ret;
2391 }
2392
2393 /* Enable the SMMU interface */
2394 enables |= CR0_SMMUEN;
2395 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2396 ARM_SMMU_CR0ACK);
2397 if (ret) {
2398 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2399 return ret;
2400 }
2401
2402 return 0;
2403}
2404
2405static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
2406{
2407 u32 reg;
2408 bool coherent;
2409 unsigned long pgsize_bitmap = 0;
2410
2411 /* IDR0 */
2412 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2413
2414 /* 2-level structures */
2415 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2416 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2417
2418 if (reg & IDR0_CD2L)
2419 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2420
2421 /*
2422 * Translation table endianness.
2423 * We currently require the same endianness as the CPU, but this
2424 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2425 */
2426 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2427 case IDR0_TTENDIAN_MIXED:
2428 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2429 break;
2430#ifdef __BIG_ENDIAN
2431 case IDR0_TTENDIAN_BE:
2432 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2433 break;
2434#else
2435 case IDR0_TTENDIAN_LE:
2436 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2437 break;
2438#endif
2439 default:
2440 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2441 return -ENXIO;
2442 }
2443
2444 /* Boolean feature flags */
2445 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2446 smmu->features |= ARM_SMMU_FEAT_PRI;
2447
2448 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2449 smmu->features |= ARM_SMMU_FEAT_ATS;
2450
2451 if (reg & IDR0_SEV)
2452 smmu->features |= ARM_SMMU_FEAT_SEV;
2453
2454 if (reg & IDR0_MSI)
2455 smmu->features |= ARM_SMMU_FEAT_MSI;
2456
2457 if (reg & IDR0_HYP)
2458 smmu->features |= ARM_SMMU_FEAT_HYP;
2459
2460 /*
2461 * The dma-coherent property is used in preference to the ID
2462 * register, but warn on mismatch.
2463 */
2464 coherent = of_dma_is_coherent(smmu->dev->of_node);
2465 if (coherent)
2466 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2467
2468 if (!!(reg & IDR0_COHACC) != coherent)
2469 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2470 coherent ? "true" : "false");
2471
Prem Mallappa6380be02015-12-14 22:01:23 +05302472 switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
2473 case IDR0_STALL_MODEL_STALL:
2474 /* Fallthrough */
2475 case IDR0_STALL_MODEL_FORCE:
Will Deacon48ec83b2015-05-27 17:25:59 +01002476 smmu->features |= ARM_SMMU_FEAT_STALLS;
Prem Mallappa6380be02015-12-14 22:01:23 +05302477 }
Will Deacon48ec83b2015-05-27 17:25:59 +01002478
2479 if (reg & IDR0_S1P)
2480 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2481
2482 if (reg & IDR0_S2P)
2483 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2484
2485 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2486 dev_err(smmu->dev, "no translation support!\n");
2487 return -ENXIO;
2488 }
2489
2490 /* We only support the AArch64 table format at present */
Will Deaconf0c453d2015-08-20 12:12:32 +01002491 switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2492 case IDR0_TTF_AARCH32_64:
2493 smmu->ias = 40;
2494 /* Fallthrough */
2495 case IDR0_TTF_AARCH64:
2496 break;
2497 default:
Will Deacon48ec83b2015-05-27 17:25:59 +01002498 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2499 return -ENXIO;
2500 }
2501
2502 /* ASID/VMID sizes */
2503 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2504 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2505
2506 /* IDR1 */
2507 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2508 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2509 dev_err(smmu->dev, "embedded implementation not supported\n");
2510 return -ENXIO;
2511 }
2512
2513 /* Queue sizes, capped at 4k */
2514 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2515 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2516 if (!smmu->cmdq.q.max_n_shift) {
2517 /* Odd alignment restrictions on the base, so ignore for now */
2518 dev_err(smmu->dev, "unit-length command queue not supported\n");
2519 return -ENXIO;
2520 }
2521
2522 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2523 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2524 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2525 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2526
2527 /* SID/SSID sizes */
2528 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2529 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2530
2531 /* IDR5 */
2532 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2533
2534 /* Maximum number of outstanding stalls */
2535 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2536 & IDR5_STALL_MAX_MASK;
2537
2538 /* Page sizes */
2539 if (reg & IDR5_GRAN64K)
2540 pgsize_bitmap |= SZ_64K | SZ_512M;
2541 if (reg & IDR5_GRAN16K)
2542 pgsize_bitmap |= SZ_16K | SZ_32M;
2543 if (reg & IDR5_GRAN4K)
2544 pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2545
2546 arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap;
2547
2548 /* Output address size */
2549 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2550 case IDR5_OAS_32_BIT:
2551 smmu->oas = 32;
2552 break;
2553 case IDR5_OAS_36_BIT:
2554 smmu->oas = 36;
2555 break;
2556 case IDR5_OAS_40_BIT:
2557 smmu->oas = 40;
2558 break;
2559 case IDR5_OAS_42_BIT:
2560 smmu->oas = 42;
2561 break;
2562 case IDR5_OAS_44_BIT:
2563 smmu->oas = 44;
2564 break;
Will Deacon85430962015-08-03 10:35:40 +01002565 default:
2566 dev_info(smmu->dev,
2567 "unknown output address size. Truncating to 48-bit\n");
2568 /* Fallthrough */
Will Deacon48ec83b2015-05-27 17:25:59 +01002569 case IDR5_OAS_48_BIT:
2570 smmu->oas = 48;
Will Deacon48ec83b2015-05-27 17:25:59 +01002571 }
2572
2573 /* Set the DMA mask for our table walker */
2574 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2575 dev_warn(smmu->dev,
2576 "failed to set DMA mask for table walker\n");
2577
Will Deaconf0c453d2015-08-20 12:12:32 +01002578 smmu->ias = max(smmu->ias, smmu->oas);
Will Deacon48ec83b2015-05-27 17:25:59 +01002579
2580 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2581 smmu->ias, smmu->oas, smmu->features);
2582 return 0;
2583}
2584
2585static int arm_smmu_device_dt_probe(struct platform_device *pdev)
2586{
2587 int irq, ret;
2588 struct resource *res;
2589 struct arm_smmu_device *smmu;
2590 struct device *dev = &pdev->dev;
2591
2592 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2593 if (!smmu) {
2594 dev_err(dev, "failed to allocate arm_smmu_device\n");
2595 return -ENOMEM;
2596 }
2597 smmu->dev = dev;
2598
2599 /* Base address */
2600 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2601 if (resource_size(res) + 1 < SZ_128K) {
2602 dev_err(dev, "MMIO region too small (%pr)\n", res);
2603 return -EINVAL;
2604 }
2605
2606 smmu->base = devm_ioremap_resource(dev, res);
2607 if (IS_ERR(smmu->base))
2608 return PTR_ERR(smmu->base);
2609
2610 /* Interrupt lines */
2611 irq = platform_get_irq_byname(pdev, "eventq");
2612 if (irq > 0)
2613 smmu->evtq.q.irq = irq;
2614
2615 irq = platform_get_irq_byname(pdev, "priq");
2616 if (irq > 0)
2617 smmu->priq.q.irq = irq;
2618
2619 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2620 if (irq > 0)
2621 smmu->cmdq.q.irq = irq;
2622
2623 irq = platform_get_irq_byname(pdev, "gerror");
2624 if (irq > 0)
2625 smmu->gerr_irq = irq;
2626
Zhen Lei5e929462015-07-07 04:30:18 +01002627 parse_driver_options(smmu);
2628
Will Deacon48ec83b2015-05-27 17:25:59 +01002629 /* Probe the h/w */
2630 ret = arm_smmu_device_probe(smmu);
2631 if (ret)
2632 return ret;
2633
2634 /* Initialise in-memory data structures */
2635 ret = arm_smmu_init_structures(smmu);
2636 if (ret)
2637 return ret;
2638
Marc Zyngier166bdbd2015-10-13 18:32:30 +01002639 /* Record our private device structure */
2640 platform_set_drvdata(pdev, smmu);
2641
Will Deacon48ec83b2015-05-27 17:25:59 +01002642 /* Reset the device */
Will Deacon04fa26c2015-10-30 18:12:41 +00002643 return arm_smmu_device_reset(smmu);
Will Deacon48ec83b2015-05-27 17:25:59 +01002644}
2645
2646static int arm_smmu_device_remove(struct platform_device *pdev)
2647{
Will Deacon941a8022015-08-11 16:25:10 +01002648 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
Will Deacon48ec83b2015-05-27 17:25:59 +01002649
2650 arm_smmu_device_disable(smmu);
Will Deacon48ec83b2015-05-27 17:25:59 +01002651 return 0;
2652}
2653
2654static struct of_device_id arm_smmu_of_match[] = {
2655 { .compatible = "arm,smmu-v3", },
2656 { },
2657};
2658MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2659
2660static struct platform_driver arm_smmu_driver = {
2661 .driver = {
2662 .name = "arm-smmu-v3",
2663 .of_match_table = of_match_ptr(arm_smmu_of_match),
2664 },
2665 .probe = arm_smmu_device_dt_probe,
2666 .remove = arm_smmu_device_remove,
2667};
2668
2669static int __init arm_smmu_init(void)
2670{
2671 struct device_node *np;
2672 int ret;
2673
2674 np = of_find_matching_node(NULL, arm_smmu_of_match);
2675 if (!np)
2676 return 0;
2677
2678 of_node_put(np);
2679
2680 ret = platform_driver_register(&arm_smmu_driver);
2681 if (ret)
2682 return ret;
2683
2684 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2685}
2686
2687static void __exit arm_smmu_exit(void)
2688{
2689 return platform_driver_unregister(&arm_smmu_driver);
2690}
2691
2692subsys_initcall(arm_smmu_init);
2693module_exit(arm_smmu_exit);
2694
2695MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2696MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2697MODULE_LICENSE("GPL v2");