blob: 33dcc29ec200c19d09c2fbe824ea05356cc62875 [file] [log] [blame]
Marek Szyprowski740a01e2016-02-18 15:12:58 +01001/*
2 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
KyongHo Cho2a965362012-05-12 05:56:09 +09003 * http://www.samsung.com
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9
10#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
11#define DEBUG
12#endif
13
KyongHo Cho2a965362012-05-12 05:56:09 +090014#include <linux/clk.h>
Marek Szyprowski8ed55c82015-05-19 15:20:36 +020015#include <linux/dma-mapping.h>
KyongHo Cho2a965362012-05-12 05:56:09 +090016#include <linux/err.h>
Marek Szyprowski312900c2015-05-19 15:20:30 +020017#include <linux/io.h>
KyongHo Cho2a965362012-05-12 05:56:09 +090018#include <linux/iommu.h>
Marek Szyprowski312900c2015-05-19 15:20:30 +020019#include <linux/interrupt.h>
KyongHo Cho2a965362012-05-12 05:56:09 +090020#include <linux/list.h>
Marek Szyprowski8ed55c82015-05-19 15:20:36 +020021#include <linux/of.h>
22#include <linux/of_iommu.h>
23#include <linux/of_platform.h>
Marek Szyprowski312900c2015-05-19 15:20:30 +020024#include <linux/platform_device.h>
25#include <linux/pm_runtime.h>
26#include <linux/slab.h>
Marek Szyprowski58c6f6a2016-02-18 15:12:49 +010027#include <linux/dma-iommu.h>
KyongHo Cho2a965362012-05-12 05:56:09 +090028
Cho KyongHod09d78f2014-05-12 11:44:58 +053029typedef u32 sysmmu_iova_t;
30typedef u32 sysmmu_pte_t;
31
Sachin Kamatf171aba2014-08-04 10:06:28 +053032/* We do not consider super section mapping (16MB) */
KyongHo Cho2a965362012-05-12 05:56:09 +090033#define SECT_ORDER 20
34#define LPAGE_ORDER 16
35#define SPAGE_ORDER 12
36
37#define SECT_SIZE (1 << SECT_ORDER)
38#define LPAGE_SIZE (1 << LPAGE_ORDER)
39#define SPAGE_SIZE (1 << SPAGE_ORDER)
40
41#define SECT_MASK (~(SECT_SIZE - 1))
42#define LPAGE_MASK (~(LPAGE_SIZE - 1))
43#define SPAGE_MASK (~(SPAGE_SIZE - 1))
44
Cho KyongHo66a7ed82014-05-12 11:45:04 +053045#define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \
46 ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
47#define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK)
48#define lv1ent_page_zero(sent) ((*(sent) & 3) == 1)
49#define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \
50 ((*(sent) & 3) == 1))
KyongHo Cho2a965362012-05-12 05:56:09 +090051#define lv1ent_section(sent) ((*(sent) & 3) == 2)
52
53#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
54#define lv2ent_small(pent) ((*(pent) & 2) == 2)
55#define lv2ent_large(pent) ((*(pent) & 3) == 1)
56
Ben Dooks6ae53432016-06-08 19:31:10 +010057#ifdef CONFIG_BIG_ENDIAN
58#warning "revisit driver if we can enable big-endian ptes"
59#endif
60
Marek Szyprowski740a01e2016-02-18 15:12:58 +010061/*
62 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
63 * v5.0 introduced support for 36bit physical address space by shifting
64 * all page entry values by 4 bits.
65 * All SYSMMU controllers in the system support the address spaces of the same
66 * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper
67 * value (0 or 4).
68 */
69static short PG_ENT_SHIFT = -1;
70#define SYSMMU_PG_ENT_SHIFT 0
71#define SYSMMU_V5_PG_ENT_SHIFT 4
KyongHo Cho2a965362012-05-12 05:56:09 +090072
Marek Szyprowski740a01e2016-02-18 15:12:58 +010073#define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT)
74#define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK)
75#define section_offs(iova) (iova & (SECT_SIZE - 1))
76#define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK)
77#define lpage_offs(iova) (iova & (LPAGE_SIZE - 1))
78#define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK)
79#define spage_offs(iova) (iova & (SPAGE_SIZE - 1))
KyongHo Cho2a965362012-05-12 05:56:09 +090080
81#define NUM_LV1ENTRIES 4096
Cho KyongHod09d78f2014-05-12 11:44:58 +053082#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
KyongHo Cho2a965362012-05-12 05:56:09 +090083
Cho KyongHod09d78f2014-05-12 11:44:58 +053084static u32 lv1ent_offset(sysmmu_iova_t iova)
85{
86 return iova >> SECT_ORDER;
87}
88
89static u32 lv2ent_offset(sysmmu_iova_t iova)
90{
91 return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1);
92}
93
Marek Szyprowski5e3435e2016-02-18 15:12:50 +010094#define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t))
Cho KyongHod09d78f2014-05-12 11:44:58 +053095#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
KyongHo Cho2a965362012-05-12 05:56:09 +090096
97#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
Marek Szyprowski740a01e2016-02-18 15:12:58 +010098#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0))
KyongHo Cho2a965362012-05-12 05:56:09 +090099
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100100#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2)
101#define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1)
102#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1)
103#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2)
KyongHo Cho2a965362012-05-12 05:56:09 +0900104
105#define CTRL_ENABLE 0x5
106#define CTRL_BLOCK 0x7
107#define CTRL_DISABLE 0x0
108
Cho KyongHoeeb51842014-05-12 11:45:03 +0530109#define CFG_LRU 0x1
110#define CFG_QOS(n) ((n & 0xF) << 7)
Cho KyongHoeeb51842014-05-12 11:45:03 +0530111#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */
112#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
113#define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */
114
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100115/* common registers */
KyongHo Cho2a965362012-05-12 05:56:09 +0900116#define REG_MMU_CTRL 0x000
117#define REG_MMU_CFG 0x004
118#define REG_MMU_STATUS 0x008
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100119#define REG_MMU_VERSION 0x034
120
121#define MMU_MAJ_VER(val) ((val) >> 7)
122#define MMU_MIN_VER(val) ((val) & 0x7F)
123#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
124
125#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
126
127/* v1.x - v3.x registers */
KyongHo Cho2a965362012-05-12 05:56:09 +0900128#define REG_MMU_FLUSH 0x00C
129#define REG_MMU_FLUSH_ENTRY 0x010
130#define REG_PT_BASE_ADDR 0x014
131#define REG_INT_STATUS 0x018
132#define REG_INT_CLEAR 0x01C
133
134#define REG_PAGE_FAULT_ADDR 0x024
135#define REG_AW_FAULT_ADDR 0x028
136#define REG_AR_FAULT_ADDR 0x02C
137#define REG_DEFAULT_SLAVE_ADDR 0x030
138
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100139/* v5.x registers */
140#define REG_V5_PT_BASE_PFN 0x00C
141#define REG_V5_MMU_FLUSH_ALL 0x010
142#define REG_V5_MMU_FLUSH_ENTRY 0x014
143#define REG_V5_INT_STATUS 0x060
144#define REG_V5_INT_CLEAR 0x064
145#define REG_V5_FAULT_AR_VA 0x070
146#define REG_V5_FAULT_AW_VA 0x080
KyongHo Cho2a965362012-05-12 05:56:09 +0900147
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530148#define has_sysmmu(dev) (dev->archdata.iommu != NULL)
149
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100150static struct device *dma_dev;
Cho KyongHo734c3c72014-05-12 11:44:48 +0530151static struct kmem_cache *lv2table_kmem_cache;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530152static sysmmu_pte_t *zero_lv2_table;
153#define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table))
Cho KyongHo734c3c72014-05-12 11:44:48 +0530154
Cho KyongHod09d78f2014-05-12 11:44:58 +0530155static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova)
KyongHo Cho2a965362012-05-12 05:56:09 +0900156{
157 return pgtable + lv1ent_offset(iova);
158}
159
Cho KyongHod09d78f2014-05-12 11:44:58 +0530160static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova)
KyongHo Cho2a965362012-05-12 05:56:09 +0900161{
Cho KyongHod09d78f2014-05-12 11:44:58 +0530162 return (sysmmu_pte_t *)phys_to_virt(
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530163 lv2table_base(sent)) + lv2ent_offset(iova);
KyongHo Cho2a965362012-05-12 05:56:09 +0900164}
165
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100166/*
167 * IOMMU fault information register
168 */
169struct sysmmu_fault_info {
170 unsigned int bit; /* bit number in STATUS register */
171 unsigned short addr_reg; /* register to read VA fault address */
172 const char *name; /* human readable fault name */
173 unsigned int type; /* fault type for report_iommu_fault */
KyongHo Cho2a965362012-05-12 05:56:09 +0900174};
175
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100176static const struct sysmmu_fault_info sysmmu_faults[] = {
177 { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ },
178 { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ },
179 { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE },
180 { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ },
181 { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ },
182 { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ },
183 { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE },
184 { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE },
KyongHo Cho2a965362012-05-12 05:56:09 +0900185};
186
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100187static const struct sysmmu_fault_info sysmmu_v5_faults[] = {
188 { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ },
189 { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ },
190 { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ },
191 { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ },
192 { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ },
193 { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE },
194 { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE },
195 { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE },
196 { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE },
197 { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE },
198};
199
Marek Szyprowski2860af32015-05-19 15:20:31 +0200200/*
201 * This structure is attached to dev.archdata.iommu of the master device
202 * on device add, contains a list of SYSMMU controllers defined by device tree,
203 * which are bound to given master device. It is usually referenced by 'owner'
204 * pointer.
205*/
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530206struct exynos_iommu_owner {
Marek Szyprowski1b092052015-05-19 15:20:33 +0200207 struct list_head controllers; /* list of sysmmu_drvdata.owner_node */
Marek Szyprowski5fa61cb2016-02-18 15:13:00 +0100208 struct iommu_domain *domain; /* domain this device is attached */
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530209};
210
Marek Szyprowski2860af32015-05-19 15:20:31 +0200211/*
212 * This structure exynos specific generalization of struct iommu_domain.
213 * It contains list of SYSMMU controllers from all master devices, which has
214 * been attached to this domain and page tables of IO address space defined by
215 * it. It is usually referenced by 'domain' pointer.
216 */
KyongHo Cho2a965362012-05-12 05:56:09 +0900217struct exynos_iommu_domain {
Marek Szyprowski2860af32015-05-19 15:20:31 +0200218 struct list_head clients; /* list of sysmmu_drvdata.domain_node */
219 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
220 short *lv2entcnt; /* free lv2 entry counter for each section */
221 spinlock_t lock; /* lock for modyfying list of clients */
222 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
Joerg Roedele1fd1ea2015-03-26 13:43:11 +0100223 struct iommu_domain domain; /* generic domain data structure */
KyongHo Cho2a965362012-05-12 05:56:09 +0900224};
225
Marek Szyprowski2860af32015-05-19 15:20:31 +0200226/*
227 * This structure hold all data of a single SYSMMU controller, this includes
228 * hw resources like registers and clocks, pointers and list nodes to connect
229 * it to all other structures, internal state and parameters read from device
230 * tree. It is usually referenced by 'data' pointer.
231 */
KyongHo Cho2a965362012-05-12 05:56:09 +0900232struct sysmmu_drvdata {
Marek Szyprowski2860af32015-05-19 15:20:31 +0200233 struct device *sysmmu; /* SYSMMU controller device */
234 struct device *master; /* master device (owner) */
235 void __iomem *sfrbase; /* our registers */
236 struct clk *clk; /* SYSMMU's clock */
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100237 struct clk *aclk; /* SYSMMU's aclk clock */
238 struct clk *pclk; /* SYSMMU's pclk clock */
Marek Szyprowski2860af32015-05-19 15:20:31 +0200239 struct clk *clk_master; /* master's device clock */
240 int activations; /* number of calls to sysmmu_enable */
241 spinlock_t lock; /* lock for modyfying state */
242 struct exynos_iommu_domain *domain; /* domain we belong to */
243 struct list_head domain_node; /* node for domain clients list */
Marek Szyprowski1b092052015-05-19 15:20:33 +0200244 struct list_head owner_node; /* node for owner controllers list */
Marek Szyprowski2860af32015-05-19 15:20:31 +0200245 phys_addr_t pgtable; /* assigned page table structure */
246 unsigned int version; /* our version */
KyongHo Cho2a965362012-05-12 05:56:09 +0900247};
248
Joerg Roedele1fd1ea2015-03-26 13:43:11 +0100249static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
250{
251 return container_of(dom, struct exynos_iommu_domain, domain);
252}
253
KyongHo Cho2a965362012-05-12 05:56:09 +0900254static bool set_sysmmu_active(struct sysmmu_drvdata *data)
255{
256 /* return true if the System MMU was not active previously
257 and it needs to be initialized */
258 return ++data->activations == 1;
259}
260
261static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
262{
263 /* return true if the System MMU is needed to be disabled */
264 BUG_ON(data->activations < 1);
265 return --data->activations == 0;
266}
267
268static bool is_sysmmu_active(struct sysmmu_drvdata *data)
269{
270 return data->activations > 0;
271}
272
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100273static void sysmmu_unblock(struct sysmmu_drvdata *data)
KyongHo Cho2a965362012-05-12 05:56:09 +0900274{
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100275 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
KyongHo Cho2a965362012-05-12 05:56:09 +0900276}
277
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100278static bool sysmmu_block(struct sysmmu_drvdata *data)
KyongHo Cho2a965362012-05-12 05:56:09 +0900279{
280 int i = 120;
281
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100282 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
283 while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1))
KyongHo Cho2a965362012-05-12 05:56:09 +0900284 --i;
285
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100286 if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) {
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100287 sysmmu_unblock(data);
KyongHo Cho2a965362012-05-12 05:56:09 +0900288 return false;
289 }
290
291 return true;
292}
293
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100294static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data)
KyongHo Cho2a965362012-05-12 05:56:09 +0900295{
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100296 if (MMU_MAJ_VER(data->version) < 5)
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100297 writel(0x1, data->sfrbase + REG_MMU_FLUSH);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100298 else
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100299 writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL);
KyongHo Cho2a965362012-05-12 05:56:09 +0900300}
301
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100302static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
Cho KyongHod09d78f2014-05-12 11:44:58 +0530303 sysmmu_iova_t iova, unsigned int num_inv)
KyongHo Cho2a965362012-05-12 05:56:09 +0900304{
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530305 unsigned int i;
Sachin Kamat365409d2014-05-22 09:50:56 +0530306
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530307 for (i = 0; i < num_inv; i++) {
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100308 if (MMU_MAJ_VER(data->version) < 5)
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100309 writel((iova & SPAGE_MASK) | 1,
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100310 data->sfrbase + REG_MMU_FLUSH_ENTRY);
311 else
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100312 writel((iova & SPAGE_MASK) | 1,
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100313 data->sfrbase + REG_V5_MMU_FLUSH_ENTRY);
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530314 iova += SPAGE_SIZE;
315 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900316}
317
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100318static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd)
KyongHo Cho2a965362012-05-12 05:56:09 +0900319{
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100320 if (MMU_MAJ_VER(data->version) < 5)
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100321 writel(pgd, data->sfrbase + REG_PT_BASE_ADDR);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100322 else
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100323 writel(pgd >> PAGE_SHIFT,
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100324 data->sfrbase + REG_V5_PT_BASE_PFN);
KyongHo Cho2a965362012-05-12 05:56:09 +0900325
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100326 __sysmmu_tlb_invalidate(data);
KyongHo Cho2a965362012-05-12 05:56:09 +0900327}
328
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200329static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data)
330{
331 BUG_ON(clk_prepare_enable(data->clk_master));
332 BUG_ON(clk_prepare_enable(data->clk));
333 BUG_ON(clk_prepare_enable(data->pclk));
334 BUG_ON(clk_prepare_enable(data->aclk));
335}
336
337static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data)
338{
339 clk_disable_unprepare(data->aclk);
340 clk_disable_unprepare(data->pclk);
341 clk_disable_unprepare(data->clk);
342 clk_disable_unprepare(data->clk_master);
343}
344
Marek Szyprowski850d3132016-02-18 15:12:56 +0100345static void __sysmmu_get_version(struct sysmmu_drvdata *data)
346{
347 u32 ver;
348
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200349 __sysmmu_enable_clocks(data);
Marek Szyprowski850d3132016-02-18 15:12:56 +0100350
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100351 ver = readl(data->sfrbase + REG_MMU_VERSION);
Marek Szyprowski850d3132016-02-18 15:12:56 +0100352
353 /* controllers on some SoCs don't report proper version */
354 if (ver == 0x80000001u)
355 data->version = MAKE_MMU_VER(1, 0);
356 else
357 data->version = MMU_RAW_VER(ver);
358
359 dev_dbg(data->sysmmu, "hardware version: %d.%d\n",
360 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version));
361
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200362 __sysmmu_disable_clocks(data);
Marek Szyprowski850d3132016-02-18 15:12:56 +0100363}
364
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100365static void show_fault_information(struct sysmmu_drvdata *data,
366 const struct sysmmu_fault_info *finfo,
367 sysmmu_iova_t fault_addr)
KyongHo Cho2a965362012-05-12 05:56:09 +0900368{
Cho KyongHod09d78f2014-05-12 11:44:58 +0530369 sysmmu_pte_t *ent;
KyongHo Cho2a965362012-05-12 05:56:09 +0900370
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100371 dev_err(data->sysmmu, "%s FAULT occurred at %#x (page table base: %pa)\n",
372 finfo->name, fault_addr, &data->pgtable);
373 ent = section_entry(phys_to_virt(data->pgtable), fault_addr);
374 dev_err(data->sysmmu, "\tLv1 entry: %#x\n", *ent);
KyongHo Cho2a965362012-05-12 05:56:09 +0900375 if (lv1ent_page(ent)) {
376 ent = page_entry(ent, fault_addr);
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100377 dev_err(data->sysmmu, "\t Lv2 entry: %#x\n", *ent);
KyongHo Cho2a965362012-05-12 05:56:09 +0900378 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900379}
380
381static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
382{
Sachin Kamatf171aba2014-08-04 10:06:28 +0530383 /* SYSMMU is in blocked state when interrupt occurred. */
KyongHo Cho2a965362012-05-12 05:56:09 +0900384 struct sysmmu_drvdata *data = dev_id;
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100385 const struct sysmmu_fault_info *finfo;
386 unsigned int i, n, itype;
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100387 sysmmu_iova_t fault_addr = -1;
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100388 unsigned short reg_status, reg_clear;
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530389 int ret = -ENOSYS;
KyongHo Cho2a965362012-05-12 05:56:09 +0900390
KyongHo Cho2a965362012-05-12 05:56:09 +0900391 WARN_ON(!is_sysmmu_active(data));
392
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100393 if (MMU_MAJ_VER(data->version) < 5) {
394 reg_status = REG_INT_STATUS;
395 reg_clear = REG_INT_CLEAR;
396 finfo = sysmmu_faults;
397 n = ARRAY_SIZE(sysmmu_faults);
398 } else {
399 reg_status = REG_V5_INT_STATUS;
400 reg_clear = REG_V5_INT_CLEAR;
401 finfo = sysmmu_v5_faults;
402 n = ARRAY_SIZE(sysmmu_v5_faults);
403 }
404
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530405 spin_lock(&data->lock);
406
Marek Szyprowskib398af22016-02-18 15:12:51 +0100407 clk_enable(data->clk_master);
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530408
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100409 itype = __ffs(readl(data->sfrbase + reg_status));
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100410 for (i = 0; i < n; i++, finfo++)
411 if (finfo->bit == itype)
412 break;
413 /* unknown/unsupported fault */
414 BUG_ON(i == n);
KyongHo Cho2a965362012-05-12 05:56:09 +0900415
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100416 /* print debug message */
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100417 fault_addr = readl(data->sfrbase + finfo->addr_reg);
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100418 show_fault_information(data, finfo, fault_addr);
KyongHo Cho2a965362012-05-12 05:56:09 +0900419
Marek Szyprowskid093fc72016-02-18 15:12:53 +0100420 if (data->domain)
421 ret = report_iommu_fault(&data->domain->domain,
422 data->master, fault_addr, finfo->type);
Cho KyongHo1fab7fa2014-05-12 11:44:56 +0530423 /* fault is not recovered by fault handler */
424 BUG_ON(ret != 0);
KyongHo Cho2a965362012-05-12 05:56:09 +0900425
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100426 writel(1 << itype, data->sfrbase + reg_clear);
Cho KyongHo1fab7fa2014-05-12 11:44:56 +0530427
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100428 sysmmu_unblock(data);
KyongHo Cho2a965362012-05-12 05:56:09 +0900429
Marek Szyprowskib398af22016-02-18 15:12:51 +0100430 clk_disable(data->clk_master);
Cho KyongHo70605872014-05-12 11:44:55 +0530431
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530432 spin_unlock(&data->lock);
KyongHo Cho2a965362012-05-12 05:56:09 +0900433
434 return IRQ_HANDLED;
435}
436
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530437static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
KyongHo Cho2a965362012-05-12 05:56:09 +0900438{
Marek Szyprowskib398af22016-02-18 15:12:51 +0100439 clk_enable(data->clk_master);
Cho KyongHo70605872014-05-12 11:44:55 +0530440
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100441 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
442 writel(0, data->sfrbase + REG_MMU_CFG);
KyongHo Cho2a965362012-05-12 05:56:09 +0900443
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200444 __sysmmu_disable_clocks(data);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530445}
KyongHo Cho2a965362012-05-12 05:56:09 +0900446
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530447static bool __sysmmu_disable(struct sysmmu_drvdata *data)
448{
449 bool disabled;
450 unsigned long flags;
451
452 spin_lock_irqsave(&data->lock, flags);
453
454 disabled = set_sysmmu_inactive(data);
455
456 if (disabled) {
457 data->pgtable = 0;
458 data->domain = NULL;
459
460 __sysmmu_disable_nocount(data);
461
462 dev_dbg(data->sysmmu, "Disabled\n");
463 } else {
464 dev_dbg(data->sysmmu, "%d times left to disable\n",
465 data->activations);
466 }
467
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530468 spin_unlock_irqrestore(&data->lock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +0900469
KyongHo Cho2a965362012-05-12 05:56:09 +0900470 return disabled;
471}
472
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530473static void __sysmmu_init_config(struct sysmmu_drvdata *data)
474{
Marek Szyprowski83addec2016-02-18 15:12:54 +0100475 unsigned int cfg;
Cho KyongHoeeb51842014-05-12 11:45:03 +0530476
Marek Szyprowski83addec2016-02-18 15:12:54 +0100477 if (data->version <= MAKE_MMU_VER(3, 1))
478 cfg = CFG_LRU | CFG_QOS(15);
479 else if (data->version <= MAKE_MMU_VER(3, 2))
480 cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL;
481 else
482 cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN;
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530483
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100484 writel(cfg, data->sfrbase + REG_MMU_CFG);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530485}
486
487static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
488{
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200489 __sysmmu_enable_clocks(data);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530490
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100491 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530492
493 __sysmmu_init_config(data);
494
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100495 __sysmmu_set_ptbase(data, data->pgtable);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530496
Marek Szyprowski84bd0422016-02-29 13:42:57 +0100497 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530498
Marek Szyprowskifecc49d2016-05-23 11:30:09 +0200499 /*
500 * SYSMMU driver keeps master's clock enabled only for the short
501 * time, while accessing the registers. For performing address
502 * translation during DMA transaction it relies on the client
503 * driver to enable it.
504 */
Marek Szyprowskib398af22016-02-18 15:12:51 +0100505 clk_disable(data->clk_master);
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530506}
507
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200508static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,
Marek Szyprowskia9133b992015-05-19 15:20:29 +0200509 struct exynos_iommu_domain *domain)
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530510{
511 int ret = 0;
512 unsigned long flags;
513
514 spin_lock_irqsave(&data->lock, flags);
515 if (set_sysmmu_active(data)) {
516 data->pgtable = pgtable;
Marek Szyprowskia9133b992015-05-19 15:20:29 +0200517 data->domain = domain;
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530518
519 __sysmmu_enable_nocount(data);
520
521 dev_dbg(data->sysmmu, "Enabled\n");
522 } else {
523 ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
524
525 dev_dbg(data->sysmmu, "already enabled\n");
526 }
527
528 if (WARN_ON(ret < 0))
529 set_sysmmu_inactive(data); /* decrement count */
530
531 spin_unlock_irqrestore(&data->lock, flags);
532
533 return ret;
534}
535
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200536static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530537 sysmmu_iova_t iova)
538{
539 unsigned long flags;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530540
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530541
542 spin_lock_irqsave(&data->lock, flags);
Marek Szyprowski01324ab2016-05-23 11:30:08 +0200543 if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) {
544 clk_enable(data->clk_master);
545 __sysmmu_tlb_invalidate_entry(data, iova, 1);
546 clk_disable(data->clk_master);
Marek Szyprowskid631ea92016-02-18 15:12:55 +0100547 }
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530548 spin_unlock_irqrestore(&data->lock, flags);
549
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530550}
551
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200552static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
553 sysmmu_iova_t iova, size_t size)
KyongHo Cho2a965362012-05-12 05:56:09 +0900554{
555 unsigned long flags;
KyongHo Cho2a965362012-05-12 05:56:09 +0900556
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530557 spin_lock_irqsave(&data->lock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +0900558 if (is_sysmmu_active(data)) {
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530559 unsigned int num_inv = 1;
Cho KyongHo70605872014-05-12 11:44:55 +0530560
Marek Szyprowskib398af22016-02-18 15:12:51 +0100561 clk_enable(data->clk_master);
Cho KyongHo70605872014-05-12 11:44:55 +0530562
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530563 /*
564 * L2TLB invalidation required
565 * 4KB page: 1 invalidation
Sachin Kamatf171aba2014-08-04 10:06:28 +0530566 * 64KB page: 16 invalidations
567 * 1MB page: 64 invalidations
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530568 * because it is set-associative TLB
569 * with 8-way and 64 sets.
570 * 1MB page can be cached in one of all sets.
571 * 64KB page can be one of 16 consecutive sets.
572 */
Marek Szyprowski512bd0c2015-05-19 15:20:24 +0200573 if (MMU_MAJ_VER(data->version) == 2)
Cho KyongHo3ad6b7f2014-05-12 11:44:49 +0530574 num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
575
Marek Szyprowski02cdc362016-02-18 15:12:52 +0100576 if (sysmmu_block(data)) {
577 __sysmmu_tlb_invalidate_entry(data, iova, num_inv);
578 sysmmu_unblock(data);
KyongHo Cho2a965362012-05-12 05:56:09 +0900579 }
Marek Szyprowskib398af22016-02-18 15:12:51 +0100580 clk_disable(data->clk_master);
KyongHo Cho2a965362012-05-12 05:56:09 +0900581 } else {
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200582 dev_dbg(data->master,
583 "disabled. Skipping TLB invalidation @ %#x\n", iova);
KyongHo Cho2a965362012-05-12 05:56:09 +0900584 }
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530585 spin_unlock_irqrestore(&data->lock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +0900586}
587
Marek Szyprowski96f66552016-05-23 13:01:27 +0200588static struct iommu_ops exynos_iommu_ops;
589
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530590static int __init exynos_sysmmu_probe(struct platform_device *pdev)
KyongHo Cho2a965362012-05-12 05:56:09 +0900591{
Cho KyongHo46c16d12014-05-12 11:44:54 +0530592 int irq, ret;
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530593 struct device *dev = &pdev->dev;
KyongHo Cho2a965362012-05-12 05:56:09 +0900594 struct sysmmu_drvdata *data;
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530595 struct resource *res;
KyongHo Cho2a965362012-05-12 05:56:09 +0900596
Cho KyongHo46c16d12014-05-12 11:44:54 +0530597 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
598 if (!data)
599 return -ENOMEM;
KyongHo Cho2a965362012-05-12 05:56:09 +0900600
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530601 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Cho KyongHo46c16d12014-05-12 11:44:54 +0530602 data->sfrbase = devm_ioremap_resource(dev, res);
603 if (IS_ERR(data->sfrbase))
604 return PTR_ERR(data->sfrbase);
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530605
Cho KyongHo46c16d12014-05-12 11:44:54 +0530606 irq = platform_get_irq(pdev, 0);
607 if (irq <= 0) {
Cho KyongHo0bf4e542014-05-12 11:45:00 +0530608 dev_err(dev, "Unable to find IRQ resource\n");
Cho KyongHo46c16d12014-05-12 11:44:54 +0530609 return irq;
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530610 }
611
Cho KyongHo46c16d12014-05-12 11:44:54 +0530612 ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0,
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530613 dev_name(dev), data);
KyongHo Cho2a965362012-05-12 05:56:09 +0900614 if (ret) {
Cho KyongHo46c16d12014-05-12 11:44:54 +0530615 dev_err(dev, "Unabled to register handler of irq %d\n", irq);
616 return ret;
KyongHo Cho2a965362012-05-12 05:56:09 +0900617 }
618
Cho KyongHo46c16d12014-05-12 11:44:54 +0530619 data->clk = devm_clk_get(dev, "sysmmu");
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200620 if (PTR_ERR(data->clk) == -ENOENT)
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100621 data->clk = NULL;
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200622 else if (IS_ERR(data->clk))
623 return PTR_ERR(data->clk);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100624
625 data->aclk = devm_clk_get(dev, "aclk");
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200626 if (PTR_ERR(data->aclk) == -ENOENT)
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100627 data->aclk = NULL;
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200628 else if (IS_ERR(data->aclk))
629 return PTR_ERR(data->aclk);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100630
631 data->pclk = devm_clk_get(dev, "pclk");
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200632 if (PTR_ERR(data->pclk) == -ENOENT)
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100633 data->pclk = NULL;
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200634 else if (IS_ERR(data->pclk))
635 return PTR_ERR(data->pclk);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100636
637 if (!data->clk && (!data->aclk || !data->pclk)) {
638 dev_err(dev, "Failed to get device clock(s)!\n");
639 return -ENOSYS;
KyongHo Cho2a965362012-05-12 05:56:09 +0900640 }
641
Cho KyongHo70605872014-05-12 11:44:55 +0530642 data->clk_master = devm_clk_get(dev, "master");
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200643 if (PTR_ERR(data->clk_master) == -ENOENT)
Marek Szyprowskib398af22016-02-18 15:12:51 +0100644 data->clk_master = NULL;
Marek Szyprowski0c2b0632016-05-23 11:30:07 +0200645 else if (IS_ERR(data->clk_master))
646 return PTR_ERR(data->clk_master);
Cho KyongHo70605872014-05-12 11:44:55 +0530647
KyongHo Cho2a965362012-05-12 05:56:09 +0900648 data->sysmmu = dev;
Cho KyongHo9d4e7a22014-05-12 11:44:57 +0530649 spin_lock_init(&data->lock);
KyongHo Cho2a965362012-05-12 05:56:09 +0900650
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530651 platform_set_drvdata(pdev, data);
652
Marek Szyprowski850d3132016-02-18 15:12:56 +0100653 __sysmmu_get_version(data);
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100654 if (PG_ENT_SHIFT < 0) {
655 if (MMU_MAJ_VER(data->version) < 5)
656 PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT;
657 else
658 PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT;
659 }
660
Cho KyongHof4723ec2014-05-12 11:44:52 +0530661 pm_runtime_enable(dev);
KyongHo Cho2a965362012-05-12 05:56:09 +0900662
Marek Szyprowski96f66552016-05-23 13:01:27 +0200663 of_iommu_set_ops(dev->of_node, &exynos_iommu_ops);
664
KyongHo Cho2a965362012-05-12 05:56:09 +0900665 return 0;
KyongHo Cho2a965362012-05-12 05:56:09 +0900666}
667
Marek Szyprowski622015e2015-05-19 15:20:35 +0200668#ifdef CONFIG_PM_SLEEP
669static int exynos_sysmmu_suspend(struct device *dev)
670{
671 struct sysmmu_drvdata *data = dev_get_drvdata(dev);
672
673 dev_dbg(dev, "suspend\n");
674 if (is_sysmmu_active(data)) {
675 __sysmmu_disable_nocount(data);
676 pm_runtime_put(dev);
677 }
678 return 0;
679}
680
681static int exynos_sysmmu_resume(struct device *dev)
682{
683 struct sysmmu_drvdata *data = dev_get_drvdata(dev);
684
685 dev_dbg(dev, "resume\n");
686 if (is_sysmmu_active(data)) {
687 pm_runtime_get_sync(dev);
688 __sysmmu_enable_nocount(data);
689 }
690 return 0;
691}
692#endif
693
694static const struct dev_pm_ops sysmmu_pm_ops = {
695 SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume)
696};
697
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530698static const struct of_device_id sysmmu_of_match[] __initconst = {
699 { .compatible = "samsung,exynos-sysmmu", },
700 { },
701};
702
703static struct platform_driver exynos_sysmmu_driver __refdata = {
704 .probe = exynos_sysmmu_probe,
705 .driver = {
KyongHo Cho2a965362012-05-12 05:56:09 +0900706 .name = "exynos-sysmmu",
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530707 .of_match_table = sysmmu_of_match,
Marek Szyprowski622015e2015-05-19 15:20:35 +0200708 .pm = &sysmmu_pm_ops,
Marek Szyprowskib54b8742016-05-20 15:48:21 +0200709 .suppress_bind_attrs = true,
KyongHo Cho2a965362012-05-12 05:56:09 +0900710 }
711};
712
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100713static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val)
KyongHo Cho2a965362012-05-12 05:56:09 +0900714{
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100715 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent),
716 DMA_TO_DEVICE);
Ben Dooks6ae53432016-06-08 19:31:10 +0100717 *ent = cpu_to_le32(val);
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100718 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent),
719 DMA_TO_DEVICE);
KyongHo Cho2a965362012-05-12 05:56:09 +0900720}
721
Joerg Roedele1fd1ea2015-03-26 13:43:11 +0100722static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
KyongHo Cho2a965362012-05-12 05:56:09 +0900723{
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200724 struct exynos_iommu_domain *domain;
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100725 dma_addr_t handle;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530726 int i;
KyongHo Cho2a965362012-05-12 05:56:09 +0900727
Marek Szyprowski740a01e2016-02-18 15:12:58 +0100728 /* Check if correct PTE offsets are initialized */
729 BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev);
KyongHo Cho2a965362012-05-12 05:56:09 +0900730
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200731 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
732 if (!domain)
Joerg Roedele1fd1ea2015-03-26 13:43:11 +0100733 return NULL;
734
Marek Szyprowski58c6f6a2016-02-18 15:12:49 +0100735 if (type == IOMMU_DOMAIN_DMA) {
736 if (iommu_get_dma_cookie(&domain->domain) != 0)
737 goto err_pgtable;
738 } else if (type != IOMMU_DOMAIN_UNMANAGED) {
739 goto err_pgtable;
740 }
741
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200742 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
743 if (!domain->pgtable)
Marek Szyprowski58c6f6a2016-02-18 15:12:49 +0100744 goto err_dma_cookie;
KyongHo Cho2a965362012-05-12 05:56:09 +0900745
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200746 domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
747 if (!domain->lv2entcnt)
KyongHo Cho2a965362012-05-12 05:56:09 +0900748 goto err_counter;
749
Sachin Kamatf171aba2014-08-04 10:06:28 +0530750 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530751 for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200752 domain->pgtable[i + 0] = ZERO_LV2LINK;
753 domain->pgtable[i + 1] = ZERO_LV2LINK;
754 domain->pgtable[i + 2] = ZERO_LV2LINK;
755 domain->pgtable[i + 3] = ZERO_LV2LINK;
756 domain->pgtable[i + 4] = ZERO_LV2LINK;
757 domain->pgtable[i + 5] = ZERO_LV2LINK;
758 domain->pgtable[i + 6] = ZERO_LV2LINK;
759 domain->pgtable[i + 7] = ZERO_LV2LINK;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530760 }
761
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100762 handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE,
763 DMA_TO_DEVICE);
764 /* For mapping page table entries we rely on dma == phys */
765 BUG_ON(handle != virt_to_phys(domain->pgtable));
KyongHo Cho2a965362012-05-12 05:56:09 +0900766
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200767 spin_lock_init(&domain->lock);
768 spin_lock_init(&domain->pgtablelock);
769 INIT_LIST_HEAD(&domain->clients);
KyongHo Cho2a965362012-05-12 05:56:09 +0900770
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200771 domain->domain.geometry.aperture_start = 0;
772 domain->domain.geometry.aperture_end = ~0UL;
773 domain->domain.geometry.force_aperture = true;
Joerg Roedel3177bb72012-07-11 12:41:10 +0200774
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200775 return &domain->domain;
KyongHo Cho2a965362012-05-12 05:56:09 +0900776
777err_counter:
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200778 free_pages((unsigned long)domain->pgtable, 2);
Marek Szyprowski58c6f6a2016-02-18 15:12:49 +0100779err_dma_cookie:
780 if (type == IOMMU_DOMAIN_DMA)
781 iommu_put_dma_cookie(&domain->domain);
KyongHo Cho2a965362012-05-12 05:56:09 +0900782err_pgtable:
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200783 kfree(domain);
Joerg Roedele1fd1ea2015-03-26 13:43:11 +0100784 return NULL;
KyongHo Cho2a965362012-05-12 05:56:09 +0900785}
786
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200787static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
KyongHo Cho2a965362012-05-12 05:56:09 +0900788{
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200789 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200790 struct sysmmu_drvdata *data, *next;
KyongHo Cho2a965362012-05-12 05:56:09 +0900791 unsigned long flags;
792 int i;
793
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200794 WARN_ON(!list_empty(&domain->clients));
KyongHo Cho2a965362012-05-12 05:56:09 +0900795
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200796 spin_lock_irqsave(&domain->lock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +0900797
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200798 list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200799 if (__sysmmu_disable(data))
800 data->master = NULL;
801 list_del_init(&data->domain_node);
KyongHo Cho2a965362012-05-12 05:56:09 +0900802 }
803
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200804 spin_unlock_irqrestore(&domain->lock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +0900805
Marek Szyprowski58c6f6a2016-02-18 15:12:49 +0100806 if (iommu_domain->type == IOMMU_DOMAIN_DMA)
807 iommu_put_dma_cookie(iommu_domain);
808
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100809 dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE,
810 DMA_TO_DEVICE);
811
KyongHo Cho2a965362012-05-12 05:56:09 +0900812 for (i = 0; i < NUM_LV1ENTRIES; i++)
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100813 if (lv1ent_page(domain->pgtable + i)) {
814 phys_addr_t base = lv2table_base(domain->pgtable + i);
815
816 dma_unmap_single(dma_dev, base, LV2TABLE_SIZE,
817 DMA_TO_DEVICE);
Cho KyongHo734c3c72014-05-12 11:44:48 +0530818 kmem_cache_free(lv2table_kmem_cache,
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100819 phys_to_virt(base));
820 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900821
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200822 free_pages((unsigned long)domain->pgtable, 2);
823 free_pages((unsigned long)domain->lv2entcnt, 1);
824 kfree(domain);
KyongHo Cho2a965362012-05-12 05:56:09 +0900825}
826
Marek Szyprowski5fa61cb2016-02-18 15:13:00 +0100827static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
828 struct device *dev)
829{
830 struct exynos_iommu_owner *owner = dev->archdata.iommu;
831 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
832 phys_addr_t pagetable = virt_to_phys(domain->pgtable);
833 struct sysmmu_drvdata *data, *next;
834 unsigned long flags;
835 bool found = false;
836
837 if (!has_sysmmu(dev) || owner->domain != iommu_domain)
838 return;
839
840 spin_lock_irqsave(&domain->lock, flags);
841 list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
842 if (data->master == dev) {
843 if (__sysmmu_disable(data)) {
844 data->master = NULL;
845 list_del_init(&data->domain_node);
846 }
847 pm_runtime_put(data->sysmmu);
848 found = true;
849 }
850 }
851 spin_unlock_irqrestore(&domain->lock, flags);
852
853 owner->domain = NULL;
854
855 if (found)
856 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
857 __func__, &pagetable);
858 else
859 dev_err(dev, "%s: No IOMMU is attached\n", __func__);
860}
861
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200862static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
KyongHo Cho2a965362012-05-12 05:56:09 +0900863 struct device *dev)
864{
Cho KyongHo6b21a5d2014-05-12 11:45:02 +0530865 struct exynos_iommu_owner *owner = dev->archdata.iommu;
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200866 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200867 struct sysmmu_drvdata *data;
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200868 phys_addr_t pagetable = virt_to_phys(domain->pgtable);
KyongHo Cho2a965362012-05-12 05:56:09 +0900869 unsigned long flags;
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200870 int ret = -ENODEV;
KyongHo Cho2a965362012-05-12 05:56:09 +0900871
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200872 if (!has_sysmmu(dev))
873 return -ENODEV;
KyongHo Cho2a965362012-05-12 05:56:09 +0900874
Marek Szyprowski5fa61cb2016-02-18 15:13:00 +0100875 if (owner->domain)
876 exynos_iommu_detach_device(owner->domain, dev);
877
Marek Szyprowski1b092052015-05-19 15:20:33 +0200878 list_for_each_entry(data, &owner->controllers, owner_node) {
Marek Szyprowskice70ca52015-05-19 15:20:34 +0200879 pm_runtime_get_sync(data->sysmmu);
Marek Szyprowskia9133b992015-05-19 15:20:29 +0200880 ret = __sysmmu_enable(data, pagetable, domain);
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200881 if (ret >= 0) {
882 data->master = dev;
KyongHo Cho2a965362012-05-12 05:56:09 +0900883
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200884 spin_lock_irqsave(&domain->lock, flags);
885 list_add_tail(&data->domain_node, &domain->clients);
886 spin_unlock_irqrestore(&domain->lock, flags);
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200887 }
888 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900889
890 if (ret < 0) {
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530891 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
892 __func__, &pagetable);
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530893 return ret;
KyongHo Cho2a965362012-05-12 05:56:09 +0900894 }
895
Marek Szyprowski5fa61cb2016-02-18 15:13:00 +0100896 owner->domain = iommu_domain;
Cho KyongHo7222e8d2014-05-12 11:44:46 +0530897 dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n",
898 __func__, &pagetable, (ret == 0) ? "" : ", again");
899
KyongHo Cho2a965362012-05-12 05:56:09 +0900900 return ret;
901}
902
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200903static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530904 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
KyongHo Cho2a965362012-05-12 05:56:09 +0900905{
Cho KyongHo61128f02014-05-12 11:44:47 +0530906 if (lv1ent_section(sent)) {
Cho KyongHod09d78f2014-05-12 11:44:58 +0530907 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova);
Cho KyongHo61128f02014-05-12 11:44:47 +0530908 return ERR_PTR(-EADDRINUSE);
909 }
910
KyongHo Cho2a965362012-05-12 05:56:09 +0900911 if (lv1ent_fault(sent)) {
Cho KyongHod09d78f2014-05-12 11:44:58 +0530912 sysmmu_pte_t *pent;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530913 bool need_flush_flpd_cache = lv1ent_zero(sent);
KyongHo Cho2a965362012-05-12 05:56:09 +0900914
Cho KyongHo734c3c72014-05-12 11:44:48 +0530915 pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC);
Arnd Bergmanndbf6c6e2016-02-29 09:45:59 +0100916 BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1));
KyongHo Cho2a965362012-05-12 05:56:09 +0900917 if (!pent)
Cho KyongHo61128f02014-05-12 11:44:47 +0530918 return ERR_PTR(-ENOMEM);
KyongHo Cho2a965362012-05-12 05:56:09 +0900919
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100920 update_pte(sent, mk_lv1ent_page(virt_to_phys(pent)));
Colin Crossdc3814f2015-05-08 17:05:44 -0700921 kmemleak_ignore(pent);
KyongHo Cho2a965362012-05-12 05:56:09 +0900922 *pgcounter = NUM_LV2ENTRIES;
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100923 dma_map_single(dma_dev, pent, LV2TABLE_SIZE, DMA_TO_DEVICE);
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530924
925 /*
Sachin Kamatf171aba2014-08-04 10:06:28 +0530926 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table,
927 * FLPD cache may cache the address of zero_l2_table. This
928 * function replaces the zero_l2_table with new L2 page table
929 * to write valid mappings.
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530930 * Accessing the valid area may cause page fault since FLPD
Sachin Kamatf171aba2014-08-04 10:06:28 +0530931 * cache may still cache zero_l2_table for the valid area
932 * instead of new L2 page table that has the mapping
933 * information of the valid area.
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530934 * Thus any replacement of zero_l2_table with other valid L2
935 * page table must involve FLPD cache invalidation for System
936 * MMU v3.3.
937 * FLPD cache invalidation is performed with TLB invalidation
938 * by VPN without blocking. It is safe to invalidate TLB without
939 * blocking because the target address of TLB invalidation is
940 * not currently mapped.
941 */
942 if (need_flush_flpd_cache) {
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200943 struct sysmmu_drvdata *data;
Sachin Kamat365409d2014-05-22 09:50:56 +0530944
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200945 spin_lock(&domain->lock);
946 list_for_each_entry(data, &domain->clients, domain_node)
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200947 sysmmu_tlb_invalidate_flpdcache(data, iova);
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200948 spin_unlock(&domain->lock);
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530949 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900950 }
951
952 return page_entry(sent, iova);
953}
954
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200955static int lv1set_section(struct exynos_iommu_domain *domain,
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530956 sysmmu_pte_t *sent, sysmmu_iova_t iova,
Cho KyongHo61128f02014-05-12 11:44:47 +0530957 phys_addr_t paddr, short *pgcnt)
KyongHo Cho2a965362012-05-12 05:56:09 +0900958{
Cho KyongHo61128f02014-05-12 11:44:47 +0530959 if (lv1ent_section(sent)) {
Cho KyongHod09d78f2014-05-12 11:44:58 +0530960 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
Cho KyongHo61128f02014-05-12 11:44:47 +0530961 iova);
KyongHo Cho2a965362012-05-12 05:56:09 +0900962 return -EADDRINUSE;
Cho KyongHo61128f02014-05-12 11:44:47 +0530963 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900964
965 if (lv1ent_page(sent)) {
Cho KyongHo61128f02014-05-12 11:44:47 +0530966 if (*pgcnt != NUM_LV2ENTRIES) {
Cho KyongHod09d78f2014-05-12 11:44:58 +0530967 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
Cho KyongHo61128f02014-05-12 11:44:47 +0530968 iova);
KyongHo Cho2a965362012-05-12 05:56:09 +0900969 return -EADDRINUSE;
Cho KyongHo61128f02014-05-12 11:44:47 +0530970 }
KyongHo Cho2a965362012-05-12 05:56:09 +0900971
Cho KyongHo734c3c72014-05-12 11:44:48 +0530972 kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0));
KyongHo Cho2a965362012-05-12 05:56:09 +0900973 *pgcnt = 0;
974 }
975
Marek Szyprowski5e3435e2016-02-18 15:12:50 +0100976 update_pte(sent, mk_lv1ent_sect(paddr));
KyongHo Cho2a965362012-05-12 05:56:09 +0900977
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200978 spin_lock(&domain->lock);
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530979 if (lv1ent_page_zero(sent)) {
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200980 struct sysmmu_drvdata *data;
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530981 /*
982 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
983 * entry by speculative prefetch of SLPD which has no mapping.
984 */
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200985 list_for_each_entry(data, &domain->clients, domain_node)
Marek Szyprowski469aceb2015-05-19 15:20:27 +0200986 sysmmu_tlb_invalidate_flpdcache(data, iova);
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530987 }
Marek Szyprowskibfa00482015-05-19 15:20:28 +0200988 spin_unlock(&domain->lock);
Cho KyongHo66a7ed82014-05-12 11:45:04 +0530989
KyongHo Cho2a965362012-05-12 05:56:09 +0900990 return 0;
991}
992
Cho KyongHod09d78f2014-05-12 11:44:58 +0530993static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
KyongHo Cho2a965362012-05-12 05:56:09 +0900994 short *pgcnt)
995{
996 if (size == SPAGE_SIZE) {
Cho KyongHo0bf4e542014-05-12 11:45:00 +0530997 if (WARN_ON(!lv2ent_fault(pent)))
KyongHo Cho2a965362012-05-12 05:56:09 +0900998 return -EADDRINUSE;
999
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001000 update_pte(pent, mk_lv2ent_spage(paddr));
KyongHo Cho2a965362012-05-12 05:56:09 +09001001 *pgcnt -= 1;
1002 } else { /* size == LPAGE_SIZE */
1003 int i;
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001004 dma_addr_t pent_base = virt_to_phys(pent);
Sachin Kamat365409d2014-05-22 09:50:56 +05301005
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001006 dma_sync_single_for_cpu(dma_dev, pent_base,
1007 sizeof(*pent) * SPAGES_PER_LPAGE,
1008 DMA_TO_DEVICE);
KyongHo Cho2a965362012-05-12 05:56:09 +09001009 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
Cho KyongHo0bf4e542014-05-12 11:45:00 +05301010 if (WARN_ON(!lv2ent_fault(pent))) {
Cho KyongHo61128f02014-05-12 11:44:47 +05301011 if (i > 0)
1012 memset(pent - i, 0, sizeof(*pent) * i);
KyongHo Cho2a965362012-05-12 05:56:09 +09001013 return -EADDRINUSE;
1014 }
1015
1016 *pent = mk_lv2ent_lpage(paddr);
1017 }
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001018 dma_sync_single_for_device(dma_dev, pent_base,
1019 sizeof(*pent) * SPAGES_PER_LPAGE,
1020 DMA_TO_DEVICE);
KyongHo Cho2a965362012-05-12 05:56:09 +09001021 *pgcnt -= SPAGES_PER_LPAGE;
1022 }
1023
1024 return 0;
1025}
1026
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301027/*
1028 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
1029 *
Sachin Kamatf171aba2014-08-04 10:06:28 +05301030 * System MMU v3.x has advanced logic to improve address translation
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301031 * performance with caching more page table entries by a page table walk.
Sachin Kamatf171aba2014-08-04 10:06:28 +05301032 * However, the logic has a bug that while caching faulty page table entries,
1033 * System MMU reports page fault if the cached fault entry is hit even though
1034 * the fault entry is updated to a valid entry after the entry is cached.
1035 * To prevent caching faulty page table entries which may be updated to valid
1036 * entries later, the virtual memory manager should care about the workaround
1037 * for the problem. The following describes the workaround.
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301038 *
1039 * Any two consecutive I/O virtual address regions must have a hole of 128KiB
Sachin Kamatf171aba2014-08-04 10:06:28 +05301040 * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug).
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301041 *
Sachin Kamatf171aba2014-08-04 10:06:28 +05301042 * Precisely, any start address of I/O virtual region must be aligned with
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301043 * the following sizes for System MMU v3.1 and v3.2.
1044 * System MMU v3.1: 128KiB
1045 * System MMU v3.2: 256KiB
1046 *
1047 * Because System MMU v3.3 caches page table entries more aggressively, it needs
Sachin Kamatf171aba2014-08-04 10:06:28 +05301048 * more workarounds.
1049 * - Any two consecutive I/O virtual regions must have a hole of size larger
1050 * than or equal to 128KiB.
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301051 * - Start address of an I/O virtual region must be aligned by 128KiB.
1052 */
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001053static int exynos_iommu_map(struct iommu_domain *iommu_domain,
1054 unsigned long l_iova, phys_addr_t paddr, size_t size,
1055 int prot)
KyongHo Cho2a965362012-05-12 05:56:09 +09001056{
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001057 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
Cho KyongHod09d78f2014-05-12 11:44:58 +05301058 sysmmu_pte_t *entry;
1059 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
KyongHo Cho2a965362012-05-12 05:56:09 +09001060 unsigned long flags;
1061 int ret = -ENOMEM;
1062
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001063 BUG_ON(domain->pgtable == NULL);
KyongHo Cho2a965362012-05-12 05:56:09 +09001064
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001065 spin_lock_irqsave(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001066
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001067 entry = section_entry(domain->pgtable, iova);
KyongHo Cho2a965362012-05-12 05:56:09 +09001068
1069 if (size == SECT_SIZE) {
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001070 ret = lv1set_section(domain, entry, iova, paddr,
1071 &domain->lv2entcnt[lv1ent_offset(iova)]);
KyongHo Cho2a965362012-05-12 05:56:09 +09001072 } else {
Cho KyongHod09d78f2014-05-12 11:44:58 +05301073 sysmmu_pte_t *pent;
KyongHo Cho2a965362012-05-12 05:56:09 +09001074
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001075 pent = alloc_lv2entry(domain, entry, iova,
1076 &domain->lv2entcnt[lv1ent_offset(iova)]);
KyongHo Cho2a965362012-05-12 05:56:09 +09001077
Cho KyongHo61128f02014-05-12 11:44:47 +05301078 if (IS_ERR(pent))
1079 ret = PTR_ERR(pent);
KyongHo Cho2a965362012-05-12 05:56:09 +09001080 else
1081 ret = lv2set_page(pent, paddr, size,
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001082 &domain->lv2entcnt[lv1ent_offset(iova)]);
KyongHo Cho2a965362012-05-12 05:56:09 +09001083 }
1084
Cho KyongHo61128f02014-05-12 11:44:47 +05301085 if (ret)
Cho KyongHo0bf4e542014-05-12 11:45:00 +05301086 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
1087 __func__, ret, size, iova);
KyongHo Cho2a965362012-05-12 05:56:09 +09001088
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001089 spin_unlock_irqrestore(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001090
1091 return ret;
1092}
1093
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001094static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain,
1095 sysmmu_iova_t iova, size_t size)
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301096{
Marek Szyprowski469aceb2015-05-19 15:20:27 +02001097 struct sysmmu_drvdata *data;
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301098 unsigned long flags;
1099
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001100 spin_lock_irqsave(&domain->lock, flags);
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301101
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001102 list_for_each_entry(data, &domain->clients, domain_node)
Marek Szyprowski469aceb2015-05-19 15:20:27 +02001103 sysmmu_tlb_invalidate_entry(data, iova, size);
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301104
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001105 spin_unlock_irqrestore(&domain->lock, flags);
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301106}
1107
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001108static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
1109 unsigned long l_iova, size_t size)
KyongHo Cho2a965362012-05-12 05:56:09 +09001110{
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001111 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
Cho KyongHod09d78f2014-05-12 11:44:58 +05301112 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
1113 sysmmu_pte_t *ent;
Cho KyongHo61128f02014-05-12 11:44:47 +05301114 size_t err_pgsize;
Cho KyongHod09d78f2014-05-12 11:44:58 +05301115 unsigned long flags;
KyongHo Cho2a965362012-05-12 05:56:09 +09001116
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001117 BUG_ON(domain->pgtable == NULL);
KyongHo Cho2a965362012-05-12 05:56:09 +09001118
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001119 spin_lock_irqsave(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001120
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001121 ent = section_entry(domain->pgtable, iova);
KyongHo Cho2a965362012-05-12 05:56:09 +09001122
1123 if (lv1ent_section(ent)) {
Cho KyongHo0bf4e542014-05-12 11:45:00 +05301124 if (WARN_ON(size < SECT_SIZE)) {
Cho KyongHo61128f02014-05-12 11:44:47 +05301125 err_pgsize = SECT_SIZE;
1126 goto err;
1127 }
KyongHo Cho2a965362012-05-12 05:56:09 +09001128
Sachin Kamatf171aba2014-08-04 10:06:28 +05301129 /* workaround for h/w bug in System MMU v3.3 */
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001130 update_pte(ent, ZERO_LV2LINK);
KyongHo Cho2a965362012-05-12 05:56:09 +09001131 size = SECT_SIZE;
1132 goto done;
1133 }
1134
1135 if (unlikely(lv1ent_fault(ent))) {
1136 if (size > SECT_SIZE)
1137 size = SECT_SIZE;
1138 goto done;
1139 }
1140
1141 /* lv1ent_page(sent) == true here */
1142
1143 ent = page_entry(ent, iova);
1144
1145 if (unlikely(lv2ent_fault(ent))) {
1146 size = SPAGE_SIZE;
1147 goto done;
1148 }
1149
1150 if (lv2ent_small(ent)) {
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001151 update_pte(ent, 0);
KyongHo Cho2a965362012-05-12 05:56:09 +09001152 size = SPAGE_SIZE;
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001153 domain->lv2entcnt[lv1ent_offset(iova)] += 1;
KyongHo Cho2a965362012-05-12 05:56:09 +09001154 goto done;
1155 }
1156
1157 /* lv1ent_large(ent) == true here */
Cho KyongHo0bf4e542014-05-12 11:45:00 +05301158 if (WARN_ON(size < LPAGE_SIZE)) {
Cho KyongHo61128f02014-05-12 11:44:47 +05301159 err_pgsize = LPAGE_SIZE;
1160 goto err;
1161 }
KyongHo Cho2a965362012-05-12 05:56:09 +09001162
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001163 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent),
1164 sizeof(*ent) * SPAGES_PER_LPAGE,
1165 DMA_TO_DEVICE);
KyongHo Cho2a965362012-05-12 05:56:09 +09001166 memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001167 dma_sync_single_for_device(dma_dev, virt_to_phys(ent),
1168 sizeof(*ent) * SPAGES_PER_LPAGE,
1169 DMA_TO_DEVICE);
KyongHo Cho2a965362012-05-12 05:56:09 +09001170 size = LPAGE_SIZE;
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001171 domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
KyongHo Cho2a965362012-05-12 05:56:09 +09001172done:
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001173 spin_unlock_irqrestore(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001174
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001175 exynos_iommu_tlb_invalidate_entry(domain, iova, size);
KyongHo Cho2a965362012-05-12 05:56:09 +09001176
KyongHo Cho2a965362012-05-12 05:56:09 +09001177 return size;
Cho KyongHo61128f02014-05-12 11:44:47 +05301178err:
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001179 spin_unlock_irqrestore(&domain->pgtablelock, flags);
Cho KyongHo61128f02014-05-12 11:44:47 +05301180
Cho KyongHo0bf4e542014-05-12 11:45:00 +05301181 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
1182 __func__, size, iova, err_pgsize);
Cho KyongHo61128f02014-05-12 11:44:47 +05301183
1184 return 0;
KyongHo Cho2a965362012-05-12 05:56:09 +09001185}
1186
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001187static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
Varun Sethibb5547ac2013-03-29 01:23:58 +05301188 dma_addr_t iova)
KyongHo Cho2a965362012-05-12 05:56:09 +09001189{
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001190 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
Cho KyongHod09d78f2014-05-12 11:44:58 +05301191 sysmmu_pte_t *entry;
KyongHo Cho2a965362012-05-12 05:56:09 +09001192 unsigned long flags;
1193 phys_addr_t phys = 0;
1194
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001195 spin_lock_irqsave(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001196
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001197 entry = section_entry(domain->pgtable, iova);
KyongHo Cho2a965362012-05-12 05:56:09 +09001198
1199 if (lv1ent_section(entry)) {
1200 phys = section_phys(entry) + section_offs(iova);
1201 } else if (lv1ent_page(entry)) {
1202 entry = page_entry(entry, iova);
1203
1204 if (lv2ent_large(entry))
1205 phys = lpage_phys(entry) + lpage_offs(iova);
1206 else if (lv2ent_small(entry))
1207 phys = spage_phys(entry) + spage_offs(iova);
1208 }
1209
Marek Szyprowskibfa00482015-05-19 15:20:28 +02001210 spin_unlock_irqrestore(&domain->pgtablelock, flags);
KyongHo Cho2a965362012-05-12 05:56:09 +09001211
1212 return phys;
1213}
1214
Marek Szyprowski6c2ae7e2016-02-18 15:12:48 +01001215static struct iommu_group *get_device_iommu_group(struct device *dev)
1216{
1217 struct iommu_group *group;
1218
1219 group = iommu_group_get(dev);
1220 if (!group)
1221 group = iommu_group_alloc();
1222
1223 return group;
1224}
1225
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301226static int exynos_iommu_add_device(struct device *dev)
1227{
1228 struct iommu_group *group;
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301229
Marek Szyprowski06801db2015-05-19 15:20:32 +02001230 if (!has_sysmmu(dev))
1231 return -ENODEV;
1232
Marek Szyprowski6c2ae7e2016-02-18 15:12:48 +01001233 group = iommu_group_get_for_dev(dev);
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301234
Marek Szyprowski6c2ae7e2016-02-18 15:12:48 +01001235 if (IS_ERR(group))
1236 return PTR_ERR(group);
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301237
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301238 iommu_group_put(group);
1239
Marek Szyprowski6c2ae7e2016-02-18 15:12:48 +01001240 return 0;
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301241}
1242
1243static void exynos_iommu_remove_device(struct device *dev)
1244{
Marek Szyprowski06801db2015-05-19 15:20:32 +02001245 if (!has_sysmmu(dev))
1246 return;
1247
Antonios Motakisbf4a1c92014-05-12 11:44:59 +05301248 iommu_group_remove_device(dev);
1249}
1250
Marek Szyprowskiaa759fd2015-05-19 15:20:37 +02001251static int exynos_iommu_of_xlate(struct device *dev,
1252 struct of_phandle_args *spec)
1253{
1254 struct exynos_iommu_owner *owner = dev->archdata.iommu;
1255 struct platform_device *sysmmu = of_find_device_by_node(spec->np);
1256 struct sysmmu_drvdata *data;
1257
1258 if (!sysmmu)
1259 return -ENODEV;
1260
1261 data = platform_get_drvdata(sysmmu);
1262 if (!data)
1263 return -ENODEV;
1264
1265 if (!owner) {
1266 owner = kzalloc(sizeof(*owner), GFP_KERNEL);
1267 if (!owner)
1268 return -ENOMEM;
1269
1270 INIT_LIST_HEAD(&owner->controllers);
1271 dev->archdata.iommu = owner;
1272 }
1273
1274 list_add_tail(&data->owner_node, &owner->controllers);
1275 return 0;
1276}
1277
Marek Szyprowski8ed55c82015-05-19 15:20:36 +02001278static struct iommu_ops exynos_iommu_ops = {
Joerg Roedele1fd1ea2015-03-26 13:43:11 +01001279 .domain_alloc = exynos_iommu_domain_alloc,
1280 .domain_free = exynos_iommu_domain_free,
Bjorn Helgaasba5fa6f2014-05-08 14:49:14 -06001281 .attach_dev = exynos_iommu_attach_device,
1282 .detach_dev = exynos_iommu_detach_device,
1283 .map = exynos_iommu_map,
1284 .unmap = exynos_iommu_unmap,
Olav Haugan315786e2014-10-25 09:55:16 -07001285 .map_sg = default_iommu_map_sg,
Bjorn Helgaasba5fa6f2014-05-08 14:49:14 -06001286 .iova_to_phys = exynos_iommu_iova_to_phys,
Marek Szyprowski6c2ae7e2016-02-18 15:12:48 +01001287 .device_group = get_device_iommu_group,
Bjorn Helgaasba5fa6f2014-05-08 14:49:14 -06001288 .add_device = exynos_iommu_add_device,
1289 .remove_device = exynos_iommu_remove_device,
KyongHo Cho2a965362012-05-12 05:56:09 +09001290 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
Marek Szyprowskiaa759fd2015-05-19 15:20:37 +02001291 .of_xlate = exynos_iommu_of_xlate,
KyongHo Cho2a965362012-05-12 05:56:09 +09001292};
1293
Marek Szyprowski8ed55c82015-05-19 15:20:36 +02001294static bool init_done;
1295
KyongHo Cho2a965362012-05-12 05:56:09 +09001296static int __init exynos_iommu_init(void)
1297{
1298 int ret;
1299
Cho KyongHo734c3c72014-05-12 11:44:48 +05301300 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
1301 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
1302 if (!lv2table_kmem_cache) {
1303 pr_err("%s: Failed to create kmem cache\n", __func__);
1304 return -ENOMEM;
1305 }
1306
KyongHo Cho2a965362012-05-12 05:56:09 +09001307 ret = platform_driver_register(&exynos_sysmmu_driver);
Cho KyongHo734c3c72014-05-12 11:44:48 +05301308 if (ret) {
1309 pr_err("%s: Failed to register driver\n", __func__);
1310 goto err_reg_driver;
1311 }
KyongHo Cho2a965362012-05-12 05:56:09 +09001312
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301313 zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
1314 if (zero_lv2_table == NULL) {
1315 pr_err("%s: Failed to allocate zero level2 page table\n",
1316 __func__);
1317 ret = -ENOMEM;
1318 goto err_zero_lv2;
1319 }
1320
Cho KyongHo734c3c72014-05-12 11:44:48 +05301321 ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
1322 if (ret) {
1323 pr_err("%s: Failed to register exynos-iommu driver.\n",
1324 __func__);
1325 goto err_set_iommu;
1326 }
KyongHo Cho2a965362012-05-12 05:56:09 +09001327
Marek Szyprowski8ed55c82015-05-19 15:20:36 +02001328 init_done = true;
1329
Cho KyongHo734c3c72014-05-12 11:44:48 +05301330 return 0;
1331err_set_iommu:
Cho KyongHo66a7ed82014-05-12 11:45:04 +05301332 kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
1333err_zero_lv2:
Cho KyongHo734c3c72014-05-12 11:44:48 +05301334 platform_driver_unregister(&exynos_sysmmu_driver);
1335err_reg_driver:
1336 kmem_cache_destroy(lv2table_kmem_cache);
KyongHo Cho2a965362012-05-12 05:56:09 +09001337 return ret;
1338}
Marek Szyprowski8ed55c82015-05-19 15:20:36 +02001339
1340static int __init exynos_iommu_of_setup(struct device_node *np)
1341{
1342 struct platform_device *pdev;
1343
1344 if (!init_done)
1345 exynos_iommu_init();
1346
1347 pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
1348 if (IS_ERR(pdev))
1349 return PTR_ERR(pdev);
1350
Marek Szyprowski5e3435e2016-02-18 15:12:50 +01001351 /*
1352 * use the first registered sysmmu device for performing
1353 * dma mapping operations on iommu page tables (cpu cache flush)
1354 */
1355 if (!dma_dev)
1356 dma_dev = &pdev->dev;
1357
Marek Szyprowski8ed55c82015-05-19 15:20:36 +02001358 return 0;
1359}
1360
1361IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
1362 exynos_iommu_of_setup);