blob: 54556713c8d161cdd2a11c1e5a0955d97e92fa38 [file] [log] [blame]
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001/*
2 * omap iommu: tlb and pagetable primitives
3 *
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -08004 * Copyright (C) 2008-2010 Nokia Corporation
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02005 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090015#include <linux/slab.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020016#include <linux/interrupt.h>
17#include <linux/ioport.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020018#include <linux/platform_device.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030019#include <linux/iommu.h>
Tony Lindgrenc8d35c82012-11-02 12:24:03 -070020#include <linux/omap-iommu.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030021#include <linux/mutex.h>
22#include <linux/spinlock.h>
Tony Lindgrened1c7de2012-11-02 12:24:06 -070023#include <linux/io.h>
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -060024#include <linux/pm_runtime.h>
Florian Vaussard3c927482014-02-28 14:42:36 -060025#include <linux/of.h>
26#include <linux/of_iommu.h>
27#include <linux/of_irq.h>
Suman Anna7d682772014-09-04 17:27:30 -050028#include <linux/of_platform.h>
Suman Anna3ca92992015-10-02 18:02:44 -050029#include <linux/regmap.h>
30#include <linux/mfd/syscon.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020031
32#include <asm/cacheflush.h>
33
Tony Lindgren2ab7c842012-11-02 12:24:14 -070034#include <linux/platform_data/iommu-omap.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020035
Ido Yariv2f7702a2012-11-02 12:24:00 -070036#include "omap-iopgtable.h"
Tony Lindgrened1c7de2012-11-02 12:24:06 -070037#include "omap-iommu.h"
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020038
Suman Anna5acc97d2014-03-17 20:31:34 -050039#define to_iommu(dev) \
40 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
41
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +020042/* bitmap of the page sizes currently supported */
43#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
44
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030045/**
46 * struct omap_iommu_domain - omap iommu domain
47 * @pgtable: the page table
48 * @iommu_dev: an omap iommu device attached to this domain. only a single
49 * iommu device can be attached for now.
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050050 * @dev: Device using this domain.
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030051 * @lock: domain lock, should be taken when attaching/detaching
52 */
53struct omap_iommu_domain {
54 u32 *pgtable;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030055 struct omap_iommu *iommu_dev;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050056 struct device *dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030057 spinlock_t lock;
Joerg Roedel8cf851e2015-03-26 13:43:09 +010058 struct iommu_domain domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030059};
60
Ido Yariv7bd9e252012-11-02 12:24:09 -070061#define MMU_LOCK_BASE_SHIFT 10
62#define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
63#define MMU_LOCK_BASE(x) \
64 ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
65
66#define MMU_LOCK_VICT_SHIFT 4
67#define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
68#define MMU_LOCK_VICT(x) \
69 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
70
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020071static struct platform_driver omap_iommu_driver;
72static struct kmem_cache *iopte_cachep;
73
74/**
Joerg Roedel8cf851e2015-03-26 13:43:09 +010075 * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
76 * @dom: generic iommu domain handle
77 **/
78static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
79{
80 return container_of(dom, struct omap_iommu_domain, domain);
81}
82
83/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030084 * omap_iommu_save_ctx - Save registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020085 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020086 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020087void omap_iommu_save_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020088{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020089 struct omap_iommu *obj = dev_to_omap_iommu(dev);
Suman Annabd4396f2014-10-22 17:22:27 -050090 u32 *p = obj->ctx;
91 int i;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020092
Suman Annabd4396f2014-10-22 17:22:27 -050093 for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
94 p[i] = iommu_read_reg(obj, i * sizeof(u32));
95 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);
96 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020097}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030098EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020099
100/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300101 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200102 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200103 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200104void omap_iommu_restore_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200105{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200106 struct omap_iommu *obj = dev_to_omap_iommu(dev);
Suman Annabd4396f2014-10-22 17:22:27 -0500107 u32 *p = obj->ctx;
108 int i;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200109
Suman Annabd4396f2014-10-22 17:22:27 -0500110 for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
111 iommu_write_reg(obj, p[i], i * sizeof(u32));
112 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);
113 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200114}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300115EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200116
Suman Anna3ca92992015-10-02 18:02:44 -0500117static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable)
118{
119 u32 val, mask;
120
121 if (!obj->syscfg)
122 return;
123
124 mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT));
125 val = enable ? mask : 0;
126 regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val);
127}
128
Suman Annabd4396f2014-10-22 17:22:27 -0500129static void __iommu_set_twl(struct omap_iommu *obj, bool on)
130{
131 u32 l = iommu_read_reg(obj, MMU_CNTL);
132
133 if (on)
134 iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);
135 else
136 iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);
137
138 l &= ~MMU_CNTL_MASK;
139 if (on)
140 l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN);
141 else
142 l |= (MMU_CNTL_MMU_EN);
143
144 iommu_write_reg(obj, l, MMU_CNTL);
145}
146
147static int omap2_iommu_enable(struct omap_iommu *obj)
148{
149 u32 l, pa;
150
151 if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K))
152 return -EINVAL;
153
154 pa = virt_to_phys(obj->iopgd);
155 if (!IS_ALIGNED(pa, SZ_16K))
156 return -EINVAL;
157
158 l = iommu_read_reg(obj, MMU_REVISION);
159 dev_info(obj->dev, "%s: version %d.%d\n", obj->name,
160 (l >> 4) & 0xf, l & 0xf);
161
162 iommu_write_reg(obj, pa, MMU_TTB);
163
Suman Anna3ca92992015-10-02 18:02:44 -0500164 dra7_cfg_dspsys_mmu(obj, true);
165
Suman Annabd4396f2014-10-22 17:22:27 -0500166 if (obj->has_bus_err_back)
167 iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
168
169 __iommu_set_twl(obj, true);
170
171 return 0;
172}
173
174static void omap2_iommu_disable(struct omap_iommu *obj)
175{
176 u32 l = iommu_read_reg(obj, MMU_CNTL);
177
178 l &= ~MMU_CNTL_MASK;
179 iommu_write_reg(obj, l, MMU_CNTL);
Suman Anna3ca92992015-10-02 18:02:44 -0500180 dra7_cfg_dspsys_mmu(obj, false);
Suman Annabd4396f2014-10-22 17:22:27 -0500181
182 dev_dbg(obj->dev, "%s is shutting down\n", obj->name);
183}
184
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300185static int iommu_enable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200186{
187 int err;
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600188 struct platform_device *pdev = to_platform_device(obj->dev);
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530189 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200190
Florian Vaussard90e569c2014-02-28 14:42:34 -0600191 if (pdata && pdata->deassert_reset) {
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600192 err = pdata->deassert_reset(pdev, pdata->reset_name);
193 if (err) {
194 dev_err(obj->dev, "deassert_reset failed: %d\n", err);
195 return err;
196 }
197 }
198
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600199 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200200
Suman Annabd4396f2014-10-22 17:22:27 -0500201 err = omap2_iommu_enable(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200202
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200203 return err;
204}
205
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300206static void iommu_disable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200207{
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600208 struct platform_device *pdev = to_platform_device(obj->dev);
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530209 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600210
Suman Annabd4396f2014-10-22 17:22:27 -0500211 omap2_iommu_disable(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200212
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600213 pm_runtime_put_sync(obj->dev);
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600214
Florian Vaussard90e569c2014-02-28 14:42:34 -0600215 if (pdata && pdata->assert_reset)
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600216 pdata->assert_reset(pdev, pdata->reset_name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200217}
218
219/*
220 * TLB operations
221 */
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300222static u32 iotlb_cr_to_virt(struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200223{
Suman Annabd4396f2014-10-22 17:22:27 -0500224 u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK;
225 u32 mask = get_cam_va_mask(cr->cam & page_size);
226
227 return cr->cam & mask;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200228}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200229
230static u32 get_iopte_attr(struct iotlb_entry *e)
231{
Suman Annabd4396f2014-10-22 17:22:27 -0500232 u32 attr;
233
234 attr = e->mixed << 5;
235 attr |= e->endian;
236 attr |= e->elsz >> 3;
237 attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) ||
238 (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6);
239 return attr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200240}
241
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300242static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200243{
Suman Annabd4396f2014-10-22 17:22:27 -0500244 u32 status, fault_addr;
245
246 status = iommu_read_reg(obj, MMU_IRQSTATUS);
247 status &= MMU_IRQ_MASK;
248 if (!status) {
249 *da = 0;
250 return 0;
251 }
252
253 fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);
254 *da = fault_addr;
255
256 iommu_write_reg(obj, status, MMU_IRQSTATUS);
257
258 return status;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200259}
260
Suman Anna69c2c192015-07-20 17:33:25 -0500261void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200262{
263 u32 val;
264
265 val = iommu_read_reg(obj, MMU_LOCK);
266
267 l->base = MMU_LOCK_BASE(val);
268 l->vict = MMU_LOCK_VICT(val);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200269}
270
Suman Anna69c2c192015-07-20 17:33:25 -0500271void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200272{
273 u32 val;
274
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200275 val = (l->base << MMU_LOCK_BASE_SHIFT);
276 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
277
278 iommu_write_reg(obj, val, MMU_LOCK);
279}
280
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300281static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200282{
Suman Annabd4396f2014-10-22 17:22:27 -0500283 cr->cam = iommu_read_reg(obj, MMU_READ_CAM);
284 cr->ram = iommu_read_reg(obj, MMU_READ_RAM);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200285}
286
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300287static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200288{
Suman Annabd4396f2014-10-22 17:22:27 -0500289 iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);
290 iommu_write_reg(obj, cr->ram, MMU_RAM);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200291
292 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
293 iommu_write_reg(obj, 1, MMU_LD_TLB);
294}
295
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000296/* only used in iotlb iteration for-loop */
Suman Anna69c2c192015-07-20 17:33:25 -0500297struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000298{
299 struct cr_regs cr;
300 struct iotlb_lock l;
301
302 iotlb_lock_get(obj, &l);
303 l.vict = n;
304 iotlb_lock_set(obj, &l);
305 iotlb_read_cr(obj, &cr);
306
307 return cr;
308}
309
Suman Annabd4396f2014-10-22 17:22:27 -0500310#ifdef PREFETCH_IOTLB
311static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
312 struct iotlb_entry *e)
313{
314 struct cr_regs *cr;
315
316 if (!e)
317 return NULL;
318
319 if (e->da & ~(get_cam_va_mask(e->pgsz))) {
320 dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,
321 e->da);
322 return ERR_PTR(-EINVAL);
323 }
324
325 cr = kmalloc(sizeof(*cr), GFP_KERNEL);
326 if (!cr)
327 return ERR_PTR(-ENOMEM);
328
329 cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;
330 cr->ram = e->pa | e->endian | e->elsz | e->mixed;
331
332 return cr;
333}
334
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200335/**
336 * load_iotlb_entry - Set an iommu tlb entry
337 * @obj: target iommu
338 * @e: an iommu tlb entry info
339 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300340static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200341{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200342 int err = 0;
343 struct iotlb_lock l;
344 struct cr_regs *cr;
345
346 if (!obj || !obj->nr_tlb_entries || !e)
347 return -EINVAL;
348
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600349 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200350
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000351 iotlb_lock_get(obj, &l);
352 if (l.base == obj->nr_tlb_entries) {
353 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200354 err = -EBUSY;
355 goto out;
356 }
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000357 if (!e->prsvd) {
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000358 int i;
359 struct cr_regs tmp;
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000360
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000361 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000362 if (!iotlb_cr_valid(&tmp))
363 break;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000364
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000365 if (i == obj->nr_tlb_entries) {
366 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
367 err = -EBUSY;
368 goto out;
369 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000370
371 iotlb_lock_get(obj, &l);
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000372 } else {
373 l.vict = l.base;
374 iotlb_lock_set(obj, &l);
375 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200376
377 cr = iotlb_alloc_cr(obj, e);
378 if (IS_ERR(cr)) {
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600379 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200380 return PTR_ERR(cr);
381 }
382
383 iotlb_load_cr(obj, cr);
384 kfree(cr);
385
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000386 if (e->prsvd)
387 l.base++;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200388 /* increment victim for next tlb load */
389 if (++l.vict == obj->nr_tlb_entries)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000390 l.vict = l.base;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200391 iotlb_lock_set(obj, &l);
392out:
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600393 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200394 return err;
395}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200396
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300397#else /* !PREFETCH_IOTLB */
398
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300399static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300400{
401 return 0;
402}
403
404#endif /* !PREFETCH_IOTLB */
405
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300406static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300407{
408 return load_iotlb_entry(obj, e);
409}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200410
411/**
412 * flush_iotlb_page - Clear an iommu tlb entry
413 * @obj: target iommu
414 * @da: iommu device virtual address
415 *
416 * Clear an iommu tlb entry which includes 'da' address.
417 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300418static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200419{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200420 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000421 struct cr_regs cr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200422
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600423 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200424
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000425 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200426 u32 start;
427 size_t bytes;
428
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200429 if (!iotlb_cr_valid(&cr))
430 continue;
431
432 start = iotlb_cr_to_virt(&cr);
433 bytes = iopgsz_to_bytes(cr.cam & 3);
434
435 if ((start <= da) && (da < start + bytes)) {
436 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
437 __func__, start, da, bytes);
Hari Kanigeri0fa035e2010-08-20 13:50:18 +0000438 iotlb_load_cr(obj, &cr);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200439 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
Laurent Pinchartf7129a02014-03-07 23:47:03 +0100440 break;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200441 }
442 }
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600443 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200444
445 if (i == obj->nr_tlb_entries)
446 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
447}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200448
449/**
450 * flush_iotlb_all - Clear all iommu tlb entries
451 * @obj: target iommu
452 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300453static void flush_iotlb_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200454{
455 struct iotlb_lock l;
456
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600457 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200458
459 l.base = 0;
460 l.vict = 0;
461 iotlb_lock_set(obj, &l);
462
463 iommu_write_reg(obj, 1, MMU_GFLUSH);
464
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600465 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200466}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200467
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200468/*
469 * H/W pagetable operations
470 */
471static void flush_iopgd_range(u32 *first, u32 *last)
472{
473 /* FIXME: L2 cache should be taken care of if it exists */
474 do {
475 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
476 : : "r" (first));
477 first += L1_CACHE_BYTES / sizeof(*first);
478 } while (first <= last);
479}
480
481static void flush_iopte_range(u32 *first, u32 *last)
482{
483 /* FIXME: L2 cache should be taken care of if it exists */
484 do {
485 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
486 : : "r" (first));
487 first += L1_CACHE_BYTES / sizeof(*first);
488 } while (first <= last);
489}
490
491static void iopte_free(u32 *iopte)
492{
493 /* Note: freed iopte's must be clean ready for re-use */
Zhouyi Zhoue28045a2014-03-05 18:20:19 +0800494 if (iopte)
495 kmem_cache_free(iopte_cachep, iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200496}
497
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300498static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200499{
500 u32 *iopte;
501
502 /* a table has already existed */
503 if (*iopgd)
504 goto pte_ready;
505
506 /*
507 * do the allocation outside the page table lock
508 */
509 spin_unlock(&obj->page_table_lock);
510 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
511 spin_lock(&obj->page_table_lock);
512
513 if (!*iopgd) {
514 if (!iopte)
515 return ERR_PTR(-ENOMEM);
516
517 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
518 flush_iopgd_range(iopgd, iopgd);
519
520 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
521 } else {
522 /* We raced, free the reduniovant table */
523 iopte_free(iopte);
524 }
525
526pte_ready:
527 iopte = iopte_offset(iopgd, da);
528
529 dev_vdbg(obj->dev,
530 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
531 __func__, da, iopgd, *iopgd, iopte, *iopte);
532
533 return iopte;
534}
535
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300536static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200537{
538 u32 *iopgd = iopgd_offset(obj, da);
539
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300540 if ((da | pa) & ~IOSECTION_MASK) {
541 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
542 __func__, da, pa, IOSECTION_SIZE);
543 return -EINVAL;
544 }
545
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200546 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
547 flush_iopgd_range(iopgd, iopgd);
548 return 0;
549}
550
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300551static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200552{
553 u32 *iopgd = iopgd_offset(obj, da);
554 int i;
555
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300556 if ((da | pa) & ~IOSUPER_MASK) {
557 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
558 __func__, da, pa, IOSUPER_SIZE);
559 return -EINVAL;
560 }
561
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200562 for (i = 0; i < 16; i++)
563 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
564 flush_iopgd_range(iopgd, iopgd + 15);
565 return 0;
566}
567
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300568static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200569{
570 u32 *iopgd = iopgd_offset(obj, da);
571 u32 *iopte = iopte_alloc(obj, iopgd, da);
572
573 if (IS_ERR(iopte))
574 return PTR_ERR(iopte);
575
576 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
577 flush_iopte_range(iopte, iopte);
578
579 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
580 __func__, da, pa, iopte, *iopte);
581
582 return 0;
583}
584
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300585static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200586{
587 u32 *iopgd = iopgd_offset(obj, da);
588 u32 *iopte = iopte_alloc(obj, iopgd, da);
589 int i;
590
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300591 if ((da | pa) & ~IOLARGE_MASK) {
592 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
593 __func__, da, pa, IOLARGE_SIZE);
594 return -EINVAL;
595 }
596
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200597 if (IS_ERR(iopte))
598 return PTR_ERR(iopte);
599
600 for (i = 0; i < 16; i++)
601 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
602 flush_iopte_range(iopte, iopte + 15);
603 return 0;
604}
605
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300606static int
607iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200608{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300609 int (*fn)(struct omap_iommu *, u32, u32, u32);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200610 u32 prot;
611 int err;
612
613 if (!obj || !e)
614 return -EINVAL;
615
616 switch (e->pgsz) {
617 case MMU_CAM_PGSZ_16M:
618 fn = iopgd_alloc_super;
619 break;
620 case MMU_CAM_PGSZ_1M:
621 fn = iopgd_alloc_section;
622 break;
623 case MMU_CAM_PGSZ_64K:
624 fn = iopte_alloc_large;
625 break;
626 case MMU_CAM_PGSZ_4K:
627 fn = iopte_alloc_page;
628 break;
629 default:
630 fn = NULL;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200631 break;
632 }
633
Suman Anna7c1ab602016-04-04 17:46:19 -0500634 if (WARN_ON(!fn))
635 return -EINVAL;
636
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200637 prot = get_iopte_attr(e);
638
639 spin_lock(&obj->page_table_lock);
640 err = fn(obj, e->da, e->pa, prot);
641 spin_unlock(&obj->page_table_lock);
642
643 return err;
644}
645
646/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300647 * omap_iopgtable_store_entry - Make an iommu pte entry
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200648 * @obj: target iommu
649 * @e: an iommu tlb entry info
650 **/
Suman Anna4899a562014-10-22 17:22:32 -0500651static int
652omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200653{
654 int err;
655
656 flush_iotlb_page(obj, e->da);
657 err = iopgtable_store_entry_core(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200658 if (!err)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300659 prefetch_iotlb_entry(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200660 return err;
661}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200662
663/**
664 * iopgtable_lookup_entry - Lookup an iommu pte entry
665 * @obj: target iommu
666 * @da: iommu device virtual address
667 * @ppgd: iommu pgd entry pointer to be returned
668 * @ppte: iommu pte entry pointer to be returned
669 **/
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300670static void
671iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200672{
673 u32 *iopgd, *iopte = NULL;
674
675 iopgd = iopgd_offset(obj, da);
676 if (!*iopgd)
677 goto out;
678
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300679 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200680 iopte = iopte_offset(iopgd, da);
681out:
682 *ppgd = iopgd;
683 *ppte = iopte;
684}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200685
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300686static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200687{
688 size_t bytes;
689 u32 *iopgd = iopgd_offset(obj, da);
690 int nent = 1;
691
692 if (!*iopgd)
693 return 0;
694
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300695 if (iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200696 int i;
697 u32 *iopte = iopte_offset(iopgd, da);
698
699 bytes = IOPTE_SIZE;
700 if (*iopte & IOPTE_LARGE) {
701 nent *= 16;
702 /* rewind to the 1st entry */
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -0800703 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200704 }
705 bytes *= nent;
706 memset(iopte, 0, nent * sizeof(*iopte));
707 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
708
709 /*
710 * do table walk to check if this table is necessary or not
711 */
712 iopte = iopte_offset(iopgd, 0);
713 for (i = 0; i < PTRS_PER_IOPTE; i++)
714 if (iopte[i])
715 goto out;
716
717 iopte_free(iopte);
718 nent = 1; /* for the next L1 entry */
719 } else {
720 bytes = IOPGD_SIZE;
Hiroshi DOYUdcc730d2009-10-22 14:46:32 -0700721 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200722 nent *= 16;
723 /* rewind to the 1st entry */
Hiroshi DOYU8d33ea52010-02-15 10:03:32 -0800724 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200725 }
726 bytes *= nent;
727 }
728 memset(iopgd, 0, nent * sizeof(*iopgd));
729 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
730out:
731 return bytes;
732}
733
734/**
735 * iopgtable_clear_entry - Remove an iommu pte entry
736 * @obj: target iommu
737 * @da: iommu device virtual address
738 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300739static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200740{
741 size_t bytes;
742
743 spin_lock(&obj->page_table_lock);
744
745 bytes = iopgtable_clear_entry_core(obj, da);
746 flush_iotlb_page(obj, da);
747
748 spin_unlock(&obj->page_table_lock);
749
750 return bytes;
751}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200752
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300753static void iopgtable_clear_entry_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200754{
755 int i;
756
757 spin_lock(&obj->page_table_lock);
758
759 for (i = 0; i < PTRS_PER_IOPGD; i++) {
760 u32 da;
761 u32 *iopgd;
762
763 da = i << IOPGD_SHIFT;
764 iopgd = iopgd_offset(obj, da);
765
766 if (!*iopgd)
767 continue;
768
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300769 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200770 iopte_free(iopte_offset(iopgd, 0));
771
772 *iopgd = 0;
773 flush_iopgd_range(iopgd, iopgd);
774 }
775
776 flush_iotlb_all(obj);
777
778 spin_unlock(&obj->page_table_lock);
779}
780
781/*
782 * Device IOMMU generic operations
783 */
784static irqreturn_t iommu_fault_handler(int irq, void *data)
785{
David Cohend594f1f2011-02-16 19:35:51 +0000786 u32 da, errs;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200787 u32 *iopgd, *iopte;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300788 struct omap_iommu *obj = data;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400789 struct iommu_domain *domain = obj->domain;
Joerg Roedel8cf851e2015-03-26 13:43:09 +0100790 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200791
Suman Anna2088ecb2014-10-22 17:22:19 -0500792 if (!omap_domain->iommu_dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200793 return IRQ_NONE;
794
David Cohend594f1f2011-02-16 19:35:51 +0000795 errs = iommu_report_fault(obj, &da);
Laurent Pinchartc56b2dd2011-05-10 16:56:46 +0200796 if (errs == 0)
797 return IRQ_HANDLED;
David Cohend594f1f2011-02-16 19:35:51 +0000798
799 /* Fault callback or TLB/PTE Dynamic loading */
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400800 if (!report_iommu_fault(domain, obj->dev, da, 0))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200801 return IRQ_HANDLED;
802
Hiroshi DOYU37b29812010-05-24 02:01:52 +0000803 iommu_disable(obj);
804
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200805 iopgd = iopgd_offset(obj, da);
806
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300807 if (!iopgd_is_table(*iopgd)) {
Suman Annab6c2e092013-05-30 18:10:59 -0500808 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
Suman Anna5835b6a2015-07-20 17:33:32 -0500809 obj->name, errs, da, iopgd, *iopgd);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200810 return IRQ_NONE;
811 }
812
813 iopte = iopte_offset(iopgd, da);
814
Suman Annab6c2e092013-05-30 18:10:59 -0500815 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
Suman Anna5835b6a2015-07-20 17:33:32 -0500816 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200817
818 return IRQ_NONE;
819}
820
821static int device_match_by_alias(struct device *dev, void *data)
822{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300823 struct omap_iommu *obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200824 const char *name = data;
825
826 pr_debug("%s: %s %s\n", __func__, obj->name, name);
827
828 return strcmp(obj->name, name) == 0;
829}
830
831/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300832 * omap_iommu_attach() - attach iommu device to an iommu domain
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200833 * @name: name of target omap iommu device
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300834 * @iopgd: page table
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200835 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200836static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200837{
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600838 int err;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200839 struct device *dev;
840 struct omap_iommu *obj;
841
Suman Anna5835b6a2015-07-20 17:33:32 -0500842 dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
843 device_match_by_alias);
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200844 if (!dev)
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600845 return ERR_PTR(-ENODEV);
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200846
847 obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200848
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300849 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200850
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300851 obj->iopgd = iopgd;
852 err = iommu_enable(obj);
853 if (err)
854 goto err_enable;
855 flush_iotlb_all(obj);
856
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300857 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200858
859 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
860 return obj;
861
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200862err_enable:
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300863 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200864 return ERR_PTR(err);
865}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200866
867/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300868 * omap_iommu_detach - release iommu device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200869 * @obj: target iommu
870 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300871static void omap_iommu_detach(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200872{
Roel Kluinacf9d462010-01-08 10:29:05 -0800873 if (!obj || IS_ERR(obj))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200874 return;
875
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300876 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200877
Suman Anna2088ecb2014-10-22 17:22:19 -0500878 iommu_disable(obj);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300879 obj->iopgd = NULL;
880
881 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200882
883 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
884}
David Cohend594f1f2011-02-16 19:35:51 +0000885
Suman Anna3ca92992015-10-02 18:02:44 -0500886static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev,
887 struct omap_iommu *obj)
888{
889 struct device_node *np = pdev->dev.of_node;
890 int ret;
891
892 if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
893 return 0;
894
895 if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) {
896 dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n");
897 return -EINVAL;
898 }
899
900 obj->syscfg =
901 syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig");
902 if (IS_ERR(obj->syscfg)) {
903 /* can fail with -EPROBE_DEFER */
904 ret = PTR_ERR(obj->syscfg);
905 return ret;
906 }
907
908 if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1,
909 &obj->id)) {
910 dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n");
911 return -EINVAL;
912 }
913
914 if (obj->id != 0 && obj->id != 1) {
915 dev_err(&pdev->dev, "invalid IOMMU instance id\n");
916 return -EINVAL;
917 }
918
919 return 0;
920}
921
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200922/*
923 * OMAP Device MMU(IOMMU) detection
924 */
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800925static int omap_iommu_probe(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200926{
927 int err = -ENODEV;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200928 int irq;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300929 struct omap_iommu *obj;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200930 struct resource *res;
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530931 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Florian Vaussard3c927482014-02-28 14:42:36 -0600932 struct device_node *of = pdev->dev.of_node;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200933
Suman Annaf129b3d2014-02-28 14:42:32 -0600934 obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200935 if (!obj)
936 return -ENOMEM;
937
Florian Vaussard3c927482014-02-28 14:42:36 -0600938 if (of) {
939 obj->name = dev_name(&pdev->dev);
940 obj->nr_tlb_entries = 32;
941 err = of_property_read_u32(of, "ti,#tlb-entries",
942 &obj->nr_tlb_entries);
943 if (err && err != -EINVAL)
944 return err;
945 if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
946 return -EINVAL;
Suman Annab148d5f2014-02-28 14:42:37 -0600947 if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
948 obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
Florian Vaussard3c927482014-02-28 14:42:36 -0600949 } else {
950 obj->nr_tlb_entries = pdata->nr_tlb_entries;
951 obj->name = pdata->name;
Florian Vaussard3c927482014-02-28 14:42:36 -0600952 }
Florian Vaussard3c927482014-02-28 14:42:36 -0600953
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200954 obj->dev = &pdev->dev;
955 obj->ctx = (void *)obj + sizeof(*obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200956
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300957 spin_lock_init(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200958 spin_lock_init(&obj->page_table_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200959
960 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600961 obj->regbase = devm_ioremap_resource(obj->dev, res);
962 if (IS_ERR(obj->regbase))
963 return PTR_ERR(obj->regbase);
Aaro Koskinenda4a0f72011-03-14 12:28:32 +0000964
Suman Anna3ca92992015-10-02 18:02:44 -0500965 err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj);
966 if (err)
967 return err;
968
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200969 irq = platform_get_irq(pdev, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600970 if (irq < 0)
971 return -ENODEV;
972
973 err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
974 dev_name(obj->dev), obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200975 if (err < 0)
Suman Annaf129b3d2014-02-28 14:42:32 -0600976 return err;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200977 platform_set_drvdata(pdev, obj);
978
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600979 pm_runtime_irq_safe(obj->dev);
980 pm_runtime_enable(obj->dev);
981
Suman Anna61c75352014-10-22 17:22:30 -0500982 omap_iommu_debugfs_add(obj);
983
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200984 dev_info(&pdev->dev, "%s registered\n", obj->name);
985 return 0;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200986}
987
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800988static int omap_iommu_remove(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200989{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300990 struct omap_iommu *obj = platform_get_drvdata(pdev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200991
Suman Anna61c75352014-10-22 17:22:30 -0500992 omap_iommu_debugfs_remove(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200993
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600994 pm_runtime_disable(obj->dev);
995
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200996 dev_info(&pdev->dev, "%s removed\n", obj->name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200997 return 0;
998}
999
Kiran Padwald943b0f2014-09-11 19:07:36 +05301000static const struct of_device_id omap_iommu_of_match[] = {
Florian Vaussard3c927482014-02-28 14:42:36 -06001001 { .compatible = "ti,omap2-iommu" },
1002 { .compatible = "ti,omap4-iommu" },
1003 { .compatible = "ti,dra7-iommu" },
Suman Anna3ca92992015-10-02 18:02:44 -05001004 { .compatible = "ti,dra7-dsp-iommu" },
Florian Vaussard3c927482014-02-28 14:42:36 -06001005 {},
1006};
Florian Vaussard3c927482014-02-28 14:42:36 -06001007
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001008static struct platform_driver omap_iommu_driver = {
1009 .probe = omap_iommu_probe,
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08001010 .remove = omap_iommu_remove,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001011 .driver = {
1012 .name = "omap-iommu",
Florian Vaussard3c927482014-02-28 14:42:36 -06001013 .of_match_table = of_match_ptr(omap_iommu_of_match),
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001014 },
1015};
1016
1017static void iopte_cachep_ctor(void *iopte)
1018{
1019 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1020}
1021
Laurent Pinchart286f6002014-03-08 00:44:38 +01001022static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001023{
1024 memset(e, 0, sizeof(*e));
1025
1026 e->da = da;
1027 e->pa = pa;
Suman Annad760e3e2014-03-17 20:31:32 -05001028 e->valid = MMU_CAM_V;
Laurent Pinchart286f6002014-03-08 00:44:38 +01001029 e->pgsz = pgsz;
1030 e->endian = MMU_RAM_ENDIAN_LITTLE;
1031 e->elsz = MMU_RAM_ELSZ_8;
1032 e->mixed = 0;
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001033
1034 return iopgsz_to_bytes(e->pgsz);
1035}
1036
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001037static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
Suman Anna5835b6a2015-07-20 17:33:32 -05001038 phys_addr_t pa, size_t bytes, int prot)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001039{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001040 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001041 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001042 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001043 struct iotlb_entry e;
1044 int omap_pgsz;
Laurent Pinchart286f6002014-03-08 00:44:38 +01001045 u32 ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001046
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001047 omap_pgsz = bytes_to_iopgsz(bytes);
1048 if (omap_pgsz < 0) {
1049 dev_err(dev, "invalid size to map: %d\n", bytes);
1050 return -EINVAL;
1051 }
1052
Joerg Roedel1d7f4492015-01-22 14:42:06 +01001053 dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001054
Laurent Pinchart286f6002014-03-08 00:44:38 +01001055 iotlb_init_entry(&e, da, pa, omap_pgsz);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001056
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001057 ret = omap_iopgtable_store_entry(oiommu, &e);
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001058 if (ret)
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001059 dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001060
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001061 return ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001062}
1063
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001064static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
Suman Anna5835b6a2015-07-20 17:33:32 -05001065 size_t size)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001066{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001067 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001068 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001069 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001070
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001071 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001072
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001073 return iopgtable_clear_entry(oiommu, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001074}
1075
1076static int
1077omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1078{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001079 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001080 struct omap_iommu *oiommu;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001081 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001082 int ret = 0;
1083
Suman Annae3f595b2014-09-04 17:27:29 -05001084 if (!arch_data || !arch_data->name) {
1085 dev_err(dev, "device doesn't have an associated iommu\n");
1086 return -EINVAL;
1087 }
1088
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001089 spin_lock(&omap_domain->lock);
1090
1091 /* only a single device is supported per domain for now */
1092 if (omap_domain->iommu_dev) {
1093 dev_err(dev, "iommu domain is already attached\n");
1094 ret = -EBUSY;
1095 goto out;
1096 }
1097
1098 /* get a handle to and enable the omap iommu */
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001099 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001100 if (IS_ERR(oiommu)) {
1101 ret = PTR_ERR(oiommu);
1102 dev_err(dev, "can't get omap iommu: %d\n", ret);
1103 goto out;
1104 }
1105
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001106 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001107 omap_domain->dev = dev;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -04001108 oiommu->domain = domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001109
1110out:
1111 spin_unlock(&omap_domain->lock);
1112 return ret;
1113}
1114
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001115static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001116 struct device *dev)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001117{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001118 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001119 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001120
1121 /* only a single device is supported per domain for now */
1122 if (omap_domain->iommu_dev != oiommu) {
1123 dev_err(dev, "invalid iommu device\n");
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001124 return;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001125 }
1126
1127 iopgtable_clear_entry_all(oiommu);
1128
1129 omap_iommu_detach(oiommu);
1130
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001131 omap_domain->iommu_dev = arch_data->iommu_dev = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001132 omap_domain->dev = NULL;
Suman Annaf24d9ad2014-10-22 17:22:33 -05001133 oiommu->domain = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001134}
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001135
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001136static void omap_iommu_detach_dev(struct iommu_domain *domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001137 struct device *dev)
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001138{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001139 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001140
1141 spin_lock(&omap_domain->lock);
1142 _omap_iommu_detach_dev(omap_domain, dev);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001143 spin_unlock(&omap_domain->lock);
1144}
1145
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001146static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001147{
1148 struct omap_iommu_domain *omap_domain;
1149
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001150 if (type != IOMMU_DOMAIN_UNMANAGED)
1151 return NULL;
1152
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001153 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
Suman Anna99ee98d2015-07-20 17:33:29 -05001154 if (!omap_domain)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001155 goto out;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001156
1157 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
Suman Anna99ee98d2015-07-20 17:33:29 -05001158 if (!omap_domain->pgtable)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001159 goto fail_nomem;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001160
1161 /*
1162 * should never fail, but please keep this around to ensure
1163 * we keep the hardware happy
1164 */
Suman Anna433c4342016-04-04 17:46:20 -05001165 if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)))
1166 goto fail_align;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001167
1168 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
1169 spin_lock_init(&omap_domain->lock);
1170
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001171 omap_domain->domain.geometry.aperture_start = 0;
1172 omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1;
1173 omap_domain->domain.geometry.force_aperture = true;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001174
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001175 return &omap_domain->domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001176
Suman Anna433c4342016-04-04 17:46:20 -05001177fail_align:
1178 kfree(omap_domain->pgtable);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001179fail_nomem:
1180 kfree(omap_domain);
1181out:
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001182 return NULL;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001183}
1184
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001185static void omap_iommu_domain_free(struct iommu_domain *domain)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001186{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001187 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001188
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001189 /*
1190 * An iommu device is still attached
1191 * (currently, only one device can be attached) ?
1192 */
1193 if (omap_domain->iommu_dev)
1194 _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
1195
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001196 kfree(omap_domain->pgtable);
1197 kfree(omap_domain);
1198}
1199
1200static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001201 dma_addr_t da)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001202{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001203 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001204 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001205 struct device *dev = oiommu->dev;
1206 u32 *pgd, *pte;
1207 phys_addr_t ret = 0;
1208
1209 iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
1210
1211 if (pte) {
1212 if (iopte_is_small(*pte))
1213 ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
1214 else if (iopte_is_large(*pte))
1215 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1216 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001217 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
Suman Anna5835b6a2015-07-20 17:33:32 -05001218 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001219 } else {
1220 if (iopgd_is_section(*pgd))
1221 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1222 else if (iopgd_is_super(*pgd))
1223 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1224 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001225 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
Suman Anna5835b6a2015-07-20 17:33:32 -05001226 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001227 }
1228
1229 return ret;
1230}
1231
Laurent Pinchart07a02032014-02-28 14:42:38 -06001232static int omap_iommu_add_device(struct device *dev)
1233{
1234 struct omap_iommu_arch_data *arch_data;
1235 struct device_node *np;
Suman Anna7d682772014-09-04 17:27:30 -05001236 struct platform_device *pdev;
Laurent Pinchart07a02032014-02-28 14:42:38 -06001237
1238 /*
1239 * Allocate the archdata iommu structure for DT-based devices.
1240 *
1241 * TODO: Simplify this when removing non-DT support completely from the
1242 * IOMMU users.
1243 */
1244 if (!dev->of_node)
1245 return 0;
1246
1247 np = of_parse_phandle(dev->of_node, "iommus", 0);
1248 if (!np)
1249 return 0;
1250
Suman Anna7d682772014-09-04 17:27:30 -05001251 pdev = of_find_device_by_node(np);
1252 if (WARN_ON(!pdev)) {
1253 of_node_put(np);
1254 return -EINVAL;
1255 }
1256
Laurent Pinchart07a02032014-02-28 14:42:38 -06001257 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
1258 if (!arch_data) {
1259 of_node_put(np);
1260 return -ENOMEM;
1261 }
1262
Suman Anna7d682772014-09-04 17:27:30 -05001263 arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL);
Laurent Pinchart07a02032014-02-28 14:42:38 -06001264 dev->archdata.iommu = arch_data;
1265
1266 of_node_put(np);
1267
1268 return 0;
1269}
1270
1271static void omap_iommu_remove_device(struct device *dev)
1272{
1273 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1274
1275 if (!dev->of_node || !arch_data)
1276 return;
1277
1278 kfree(arch_data->name);
1279 kfree(arch_data);
1280}
1281
Thierry Redingb22f6432014-06-27 09:03:12 +02001282static const struct iommu_ops omap_iommu_ops = {
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001283 .domain_alloc = omap_iommu_domain_alloc,
1284 .domain_free = omap_iommu_domain_free,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001285 .attach_dev = omap_iommu_attach_dev,
1286 .detach_dev = omap_iommu_detach_dev,
1287 .map = omap_iommu_map,
1288 .unmap = omap_iommu_unmap,
Olav Haugan315786e2014-10-25 09:55:16 -07001289 .map_sg = default_iommu_map_sg,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001290 .iova_to_phys = omap_iommu_iova_to_phys,
Laurent Pinchart07a02032014-02-28 14:42:38 -06001291 .add_device = omap_iommu_add_device,
1292 .remove_device = omap_iommu_remove_device,
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +02001293 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001294};
1295
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001296static int __init omap_iommu_init(void)
1297{
1298 struct kmem_cache *p;
1299 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1300 size_t align = 1 << 10; /* L2 pagetable alignement */
Thierry Redingf938aab2015-02-06 11:44:06 +01001301 struct device_node *np;
Suman Annaa3a9a972017-04-12 00:21:26 -05001302 int ret;
Thierry Redingf938aab2015-02-06 11:44:06 +01001303
1304 np = of_find_matching_node(NULL, omap_iommu_of_match);
1305 if (!np)
1306 return 0;
1307
1308 of_node_put(np);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001309
1310 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1311 iopte_cachep_ctor);
1312 if (!p)
1313 return -ENOMEM;
1314 iopte_cachep = p;
1315
Suman Anna61c75352014-10-22 17:22:30 -05001316 omap_iommu_debugfs_init();
1317
Suman Annaa3a9a972017-04-12 00:21:26 -05001318 ret = platform_driver_register(&omap_iommu_driver);
1319 if (ret) {
1320 pr_err("%s: failed to register driver\n", __func__);
1321 goto fail_driver;
1322 }
1323
1324 ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops);
1325 if (ret)
1326 goto fail_bus;
1327
1328 return 0;
1329
1330fail_bus:
1331 platform_driver_unregister(&omap_iommu_driver);
1332fail_driver:
1333 kmem_cache_destroy(iopte_cachep);
1334 return ret;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001335}
Ohad Ben-Cohen435792d2012-02-26 12:14:14 +02001336subsys_initcall(omap_iommu_init);
Suman Anna0cdbf722015-07-20 17:33:24 -05001337/* must be ready before omap3isp is probed */