blob: 5f5a02b43d1816b58f87fd0bc94a06a13510a0d8 [file] [log] [blame]
Laura Abbott9ac7af62012-04-17 11:22:35 -07001/* Copyright (c) 2011-2012, Code Aurora Forum. All rights reserved.
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 */
12
13#include <linux/io.h>
14#include <linux/types.h>
15#include <linux/err.h>
16#include <linux/slab.h>
17#include <linux/memory_alloc.h>
18#include <linux/module.h>
19#include <mach/iommu.h>
20#include <mach/iommu_domains.h>
21#include <mach/msm_subsystem_map.h>
22
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070023struct msm_buffer_node {
24 struct rb_node rb_node_all_buffer;
25 struct rb_node rb_node_paddr;
26 struct msm_mapped_buffer *buf;
27 unsigned long length;
28 unsigned int *subsystems;
29 unsigned int nsubsys;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070030 unsigned int phys;
31};
32
33static struct rb_root buffer_root;
34static struct rb_root phys_root;
35DEFINE_MUTEX(msm_buffer_mutex);
36
Laura Abbott9ac7af62012-04-17 11:22:35 -070037static unsigned long subsystem_to_domain_tbl[] = {
38 VIDEO_DOMAIN,
39 VIDEO_DOMAIN,
40 CAMERA_DOMAIN,
Olav Hauganef95ae32012-05-15 09:50:30 -070041 DISPLAY_READ_DOMAIN,
42 DISPLAY_WRITE_DOMAIN,
43 ROTATOR_SRC_DOMAIN,
44 ROTATOR_DST_DOMAIN,
Laura Abbott9ac7af62012-04-17 11:22:35 -070045 0xFFFFFFFF
46};
47
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070048static struct msm_buffer_node *find_buffer(void *key)
49{
50 struct rb_root *root = &buffer_root;
51 struct rb_node *p = root->rb_node;
52
53 mutex_lock(&msm_buffer_mutex);
54
55 while (p) {
56 struct msm_buffer_node *node;
57
58 node = rb_entry(p, struct msm_buffer_node, rb_node_all_buffer);
59 if (node->buf->vaddr) {
60 if (key < node->buf->vaddr)
61 p = p->rb_left;
62 else if (key > node->buf->vaddr)
63 p = p->rb_right;
64 else {
65 mutex_unlock(&msm_buffer_mutex);
66 return node;
67 }
68 } else {
69 if (key < (void *)node->buf)
70 p = p->rb_left;
71 else if (key > (void *)node->buf)
72 p = p->rb_right;
73 else {
74 mutex_unlock(&msm_buffer_mutex);
75 return node;
76 }
77 }
78 }
79 mutex_unlock(&msm_buffer_mutex);
80 return NULL;
81}
82
83static struct msm_buffer_node *find_buffer_phys(unsigned int phys)
84{
85 struct rb_root *root = &phys_root;
86 struct rb_node *p = root->rb_node;
87
88 mutex_lock(&msm_buffer_mutex);
89
90 while (p) {
91 struct msm_buffer_node *node;
92
93 node = rb_entry(p, struct msm_buffer_node, rb_node_paddr);
94 if (phys < node->phys)
95 p = p->rb_left;
96 else if (phys > node->phys)
97 p = p->rb_right;
98 else {
99 mutex_unlock(&msm_buffer_mutex);
100 return node;
101 }
102 }
103 mutex_unlock(&msm_buffer_mutex);
104 return NULL;
105
106}
107
108static int add_buffer(struct msm_buffer_node *node)
109{
110 struct rb_root *root = &buffer_root;
111 struct rb_node **p = &root->rb_node;
112 struct rb_node *parent = NULL;
113 void *key;
114
115 if (node->buf->vaddr)
116 key = node->buf->vaddr;
117 else
118 key = node->buf;
119
120 mutex_lock(&msm_buffer_mutex);
121 while (*p) {
122 struct msm_buffer_node *tmp;
123 parent = *p;
124
125 tmp = rb_entry(parent, struct msm_buffer_node,
126 rb_node_all_buffer);
127
128 if (tmp->buf->vaddr) {
129 if (key < tmp->buf->vaddr)
130 p = &(*p)->rb_left;
131 else if (key > tmp->buf->vaddr)
132 p = &(*p)->rb_right;
133 else {
134 WARN(1, "tried to add buffer twice! buf = %p"
135 " vaddr = %p iova = %p", tmp->buf,
136 tmp->buf->vaddr,
137 tmp->buf->iova);
138 mutex_unlock(&msm_buffer_mutex);
139 return -EINVAL;
140
141 }
142 } else {
143 if (key < (void *)tmp->buf)
144 p = &(*p)->rb_left;
145 else if (key > (void *)tmp->buf)
146 p = &(*p)->rb_right;
147 else {
148 WARN(1, "tried to add buffer twice! buf = %p"
149 " vaddr = %p iova = %p", tmp->buf,
150 tmp->buf->vaddr,
151 tmp->buf->iova);
152 mutex_unlock(&msm_buffer_mutex);
153 return -EINVAL;
154 }
155 }
156 }
157 rb_link_node(&node->rb_node_all_buffer, parent, p);
158 rb_insert_color(&node->rb_node_all_buffer, root);
159 mutex_unlock(&msm_buffer_mutex);
160 return 0;
161}
162
163static int add_buffer_phys(struct msm_buffer_node *node)
164{
165 struct rb_root *root = &phys_root;
166 struct rb_node **p = &root->rb_node;
167 struct rb_node *parent = NULL;
168
169 mutex_lock(&msm_buffer_mutex);
170 while (*p) {
171 struct msm_buffer_node *tmp;
172 parent = *p;
173
174 tmp = rb_entry(parent, struct msm_buffer_node, rb_node_paddr);
175
176 if (node->phys < tmp->phys)
177 p = &(*p)->rb_left;
178 else if (node->phys > tmp->phys)
179 p = &(*p)->rb_right;
180 else {
181 WARN(1, "tried to add buffer twice! buf = %p"
182 " vaddr = %p iova = %p", tmp->buf,
183 tmp->buf->vaddr,
184 tmp->buf->iova);
185 mutex_unlock(&msm_buffer_mutex);
186 return -EINVAL;
187
188 }
189 }
190 rb_link_node(&node->rb_node_paddr, parent, p);
191 rb_insert_color(&node->rb_node_paddr, root);
192 mutex_unlock(&msm_buffer_mutex);
193 return 0;
194}
195
196static int remove_buffer(struct msm_buffer_node *victim_node)
197{
198 struct rb_root *root = &buffer_root;
199
200 if (!victim_node)
201 return -EINVAL;
202
203 mutex_lock(&msm_buffer_mutex);
204 rb_erase(&victim_node->rb_node_all_buffer, root);
205 mutex_unlock(&msm_buffer_mutex);
206 return 0;
207}
208
209static int remove_buffer_phys(struct msm_buffer_node *victim_node)
210{
211 struct rb_root *root = &phys_root;
212
213 if (!victim_node)
214 return -EINVAL;
215
216 mutex_lock(&msm_buffer_mutex);
217 rb_erase(&victim_node->rb_node_paddr, root);
218 mutex_unlock(&msm_buffer_mutex);
219 return 0;
220}
221
Laura Abbott9ac7af62012-04-17 11:22:35 -0700222static unsigned long msm_subsystem_get_domain_no(int subsys_id)
223{
224 if (subsys_id > INVALID_SUBSYS_ID && subsys_id <= MAX_SUBSYSTEM_ID &&
225 subsys_id < ARRAY_SIZE(subsystem_to_domain_tbl))
226 return subsystem_to_domain_tbl[subsys_id];
227 else
228 return subsystem_to_domain_tbl[MAX_SUBSYSTEM_ID];
229}
230
231static unsigned long msm_subsystem_get_partition_no(int subsys_id)
232{
233 switch (subsys_id) {
234 case MSM_SUBSYSTEM_VIDEO_FWARE:
235 return VIDEO_FIRMWARE_POOL;
236 case MSM_SUBSYSTEM_VIDEO:
237 return VIDEO_MAIN_POOL;
238 case MSM_SUBSYSTEM_CAMERA:
239 case MSM_SUBSYSTEM_DISPLAY:
240 case MSM_SUBSYSTEM_ROTATOR:
241 return GEN_POOL;
242 default:
243 return 0xFFFFFFFF;
244 }
245}
246
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700247phys_addr_t msm_subsystem_check_iova_mapping(int subsys_id, unsigned long iova)
248{
249 struct iommu_domain *subsys_domain;
250
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700251 if (!msm_use_iommu())
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700252 /*
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700253 * If there is no iommu, Just return the iova in this case.
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700254 */
255 return iova;
256
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700257 subsys_domain = msm_get_iommu_domain(msm_subsystem_get_domain_no
258 (subsys_id));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700259
260 return iommu_iova_to_phys(subsys_domain, iova);
261}
262EXPORT_SYMBOL(msm_subsystem_check_iova_mapping);
263
264struct msm_mapped_buffer *msm_subsystem_map_buffer(unsigned long phys,
265 unsigned int length,
266 unsigned int flags,
267 int *subsys_ids,
268 unsigned int nsubsys)
269{
270 struct msm_mapped_buffer *buf, *err;
271 struct msm_buffer_node *node;
272 int i = 0, j = 0, ret;
273 unsigned long iova_start = 0, temp_phys, temp_va = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700274 struct iommu_domain *d = NULL;
Laura Abbotte956cce2011-10-25 13:33:20 -0700275 int map_size = length;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700276
277 if (!((flags & MSM_SUBSYSTEM_MAP_KADDR) ||
278 (flags & MSM_SUBSYSTEM_MAP_IOVA))) {
279 pr_warn("%s: no mapping flag was specified. The caller"
280 " should explicitly specify what to map in the"
281 " flags.\n", __func__);
282 err = ERR_PTR(-EINVAL);
283 goto outret;
284 }
285
286 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
287 if (!buf) {
288 err = ERR_PTR(-ENOMEM);
289 goto outret;
290 }
291
292 node = kzalloc(sizeof(*node), GFP_ATOMIC);
293 if (!node) {
294 err = ERR_PTR(-ENOMEM);
295 goto outkfreebuf;
296 }
297
298 node->phys = phys;
299
300 if (flags & MSM_SUBSYSTEM_MAP_KADDR) {
301 struct msm_buffer_node *old_buffer;
302
303 old_buffer = find_buffer_phys(phys);
304
305 if (old_buffer) {
306 WARN(1, "%s: Attempting to map %lx twice in the kernel"
307 " virtual space. Don't do that!\n", __func__,
308 phys);
309 err = ERR_PTR(-EINVAL);
310 goto outkfreenode;
311 }
312
313 if (flags & MSM_SUBSYSTEM_MAP_CACHED)
314 buf->vaddr = ioremap(phys, length);
315 else if (flags & MSM_SUBSYSTEM_MAP_KADDR)
316 buf->vaddr = ioremap_nocache(phys, length);
317 else {
318 pr_warn("%s: no cachability flag was indicated. Caller"
319 " must specify a cachability flag.\n",
320 __func__);
321 err = ERR_PTR(-EINVAL);
322 goto outkfreenode;
323 }
324
325 if (!buf->vaddr) {
326 pr_err("%s: could not ioremap\n", __func__);
327 err = ERR_PTR(-EINVAL);
328 goto outkfreenode;
329 }
330
331 if (add_buffer_phys(node)) {
332 err = ERR_PTR(-EINVAL);
333 goto outiounmap;
334 }
335 }
336
337 if ((flags & MSM_SUBSYSTEM_MAP_IOVA) && subsys_ids) {
Laura Abbott11962582011-08-02 16:29:21 -0700338 int min_align;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700339
Laura Abbott11962582011-08-02 16:29:21 -0700340 length = round_up(length, SZ_4K);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700341
Laura Abbotte956cce2011-10-25 13:33:20 -0700342 if (flags & MSM_SUBSYSTEM_MAP_IOMMU_2X)
343 map_size = 2 * length;
344 else
345 map_size = length;
346
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700347 buf->iova = kzalloc(sizeof(unsigned long)*nsubsys, GFP_ATOMIC);
348 if (!buf->iova) {
349 err = ERR_PTR(-ENOMEM);
350 goto outremovephys;
351 }
352
Laura Abbott675b31f2011-07-19 10:37:43 -0700353 /*
354 * The alignment must be specified as the exact value wanted
355 * e.g. 8k alignment must pass (0x2000 | other flags)
356 */
357 min_align = flags & ~(SZ_4K - 1);
358
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700359 for (i = 0; i < nsubsys; i++) {
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700360 unsigned int domain_no, partition_no;
361
362 if (!msm_use_iommu()) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700363 buf->iova[i] = phys;
364 continue;
365 }
366
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700367 d = msm_get_iommu_domain(
368 msm_subsystem_get_domain_no(subsys_ids[i]));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700369
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700370 if (!d) {
371 pr_err("%s: could not get domain for subsystem"
372 " %d\n", __func__, subsys_ids[i]);
373 continue;
374 }
375
376 domain_no = msm_subsystem_get_domain_no(subsys_ids[i]);
377 partition_no = msm_subsystem_get_partition_no(
378 subsys_ids[i]);
379
Laura Abbottd01221b2012-05-16 17:52:49 -0700380 ret = msm_allocate_iova_address(domain_no,
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700381 partition_no,
Laura Abbotte956cce2011-10-25 13:33:20 -0700382 map_size,
Laura Abbottd01221b2012-05-16 17:52:49 -0700383 max(min_align, SZ_4K),
384 &iova_start);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700385
Laura Abbottd01221b2012-05-16 17:52:49 -0700386 if (ret) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700387 pr_err("%s: could not allocate iova address\n",
388 __func__);
389 continue;
390 }
391
392 temp_phys = phys;
393 temp_va = iova_start;
Laura Abbott11962582011-08-02 16:29:21 -0700394 for (j = length; j > 0; j -= SZ_4K,
395 temp_phys += SZ_4K,
396 temp_va += SZ_4K) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700397 ret = iommu_map(d, temp_va, temp_phys,
Steve Mucklef132c6c2012-06-06 18:30:57 -0700398 SZ_4K,
Olav Hauganf310cf22012-05-08 08:42:49 -0700399 (IOMMU_READ | IOMMU_WRITE));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700400 if (ret) {
401 pr_err("%s: could not map iommu for"
402 " domain %p, iova %lx,"
403 " phys %lx\n", __func__, d,
404 temp_va, temp_phys);
405 err = ERR_PTR(-EINVAL);
406 goto outdomain;
407 }
408 }
409 buf->iova[i] = iova_start;
Laura Abbotte956cce2011-10-25 13:33:20 -0700410
411 if (flags & MSM_SUBSYSTEM_MAP_IOMMU_2X)
412 msm_iommu_map_extra
Olav Haugan8726caf2012-05-10 15:11:35 -0700413 (d, temp_va, length, SZ_4K,
Olav Hauganf310cf22012-05-08 08:42:49 -0700414 (IOMMU_READ | IOMMU_WRITE));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700415 }
416
417 }
418
419 node->buf = buf;
420 node->subsystems = subsys_ids;
Laura Abbotte956cce2011-10-25 13:33:20 -0700421 node->length = map_size;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700422 node->nsubsys = nsubsys;
423
424 if (add_buffer(node)) {
425 err = ERR_PTR(-EINVAL);
426 goto outiova;
427 }
428
429 return buf;
430
431outiova:
432 if (flags & MSM_SUBSYSTEM_MAP_IOVA)
Steve Mucklef132c6c2012-06-06 18:30:57 -0700433 iommu_unmap(d, temp_va, SZ_4K);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700434outdomain:
435 if (flags & MSM_SUBSYSTEM_MAP_IOVA) {
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700436 /* Unmap the rest of the current domain, i */
Laura Abbott11962582011-08-02 16:29:21 -0700437 for (j -= SZ_4K, temp_va -= SZ_4K;
438 j > 0; temp_va -= SZ_4K, j -= SZ_4K)
Steve Mucklef132c6c2012-06-06 18:30:57 -0700439 iommu_unmap(d, temp_va, SZ_4K);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700440
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700441 /* Unmap all the other domains */
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700442 for (i--; i >= 0; i--) {
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700443 unsigned int domain_no, partition_no;
444 if (!msm_use_iommu())
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700445 continue;
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700446 domain_no = msm_subsystem_get_domain_no(subsys_ids[i]);
447 partition_no = msm_subsystem_get_partition_no(
448 subsys_ids[i]);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700449
450 temp_va = buf->iova[i];
Laura Abbott11962582011-08-02 16:29:21 -0700451 for (j = length; j > 0; j -= SZ_4K,
452 temp_va += SZ_4K)
Steve Mucklef132c6c2012-06-06 18:30:57 -0700453 iommu_unmap(d, temp_va, SZ_4K);
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700454 msm_free_iova_address(buf->iova[i], domain_no,
455 partition_no, length);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700456 }
457
458 kfree(buf->iova);
459 }
460
461outremovephys:
462 if (flags & MSM_SUBSYSTEM_MAP_KADDR)
463 remove_buffer_phys(node);
464outiounmap:
465 if (flags & MSM_SUBSYSTEM_MAP_KADDR)
466 iounmap(buf->vaddr);
467outkfreenode:
468 kfree(node);
469outkfreebuf:
470 kfree(buf);
471outret:
472 return err;
473}
474EXPORT_SYMBOL(msm_subsystem_map_buffer);
475
476int msm_subsystem_unmap_buffer(struct msm_mapped_buffer *buf)
477{
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700478 struct msm_buffer_node *node;
479 int i, j, ret;
480 unsigned long temp_va;
481
Laura Abbotta7f16e32011-11-30 10:00:18 -0800482 if (IS_ERR_OR_NULL(buf))
483 goto out;
484
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700485 if (buf->vaddr)
486 node = find_buffer(buf->vaddr);
487 else
488 node = find_buffer(buf);
489
490 if (!node)
491 goto out;
492
493 if (node->buf != buf) {
494 pr_err("%s: caller must pass in the same buffer structure"
495 " returned from map_buffer when freeding\n", __func__);
496 goto out;
497 }
498
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700499 if (buf->iova) {
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700500 if (msm_use_iommu())
501 for (i = 0; i < node->nsubsys; i++) {
502 struct iommu_domain *subsys_domain;
503 unsigned int domain_no, partition_no;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700504
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700505 subsys_domain = msm_get_iommu_domain(
506 msm_subsystem_get_domain_no(
507 node->subsystems[i]));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700508
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700509 domain_no = msm_subsystem_get_domain_no(
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700510 node->subsystems[i]);
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700511 partition_no = msm_subsystem_get_partition_no(
512 node->subsystems[i]);
513
514 temp_va = buf->iova[i];
515 for (j = node->length; j > 0; j -= SZ_4K,
516 temp_va += SZ_4K) {
517 ret = iommu_unmap(subsys_domain,
518 temp_va,
Steve Mucklef132c6c2012-06-06 18:30:57 -0700519 SZ_4K);
Laura Abbott9f4a8e62011-08-29 19:08:07 -0700520 WARN(ret, "iommu_unmap returned a "
521 " non-zero value.\n");
522 }
523 msm_free_iova_address(buf->iova[i], domain_no,
524 partition_no, node->length);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700525 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700526 kfree(buf->iova);
527
528 }
529
530 if (buf->vaddr) {
531 remove_buffer_phys(node);
532 iounmap(buf->vaddr);
533 }
534
535 remove_buffer(node);
536 kfree(node);
537 kfree(buf);
538
539 return 0;
540out:
541 return -EINVAL;
542}
543EXPORT_SYMBOL(msm_subsystem_unmap_buffer);