blob: 1a022dc4188e63744948c60a3ac0e91aeee8babd [file] [log] [blame]
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001/*
2 * Copyright (C) 2012 Samsung Electronics Co.Ltd
3 * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundationr
8 */
9
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/clk.h>
13#include <linux/err.h>
14#include <linux/interrupt.h>
15#include <linux/io.h>
16#include <linux/platform_device.h>
17#include <linux/pm_runtime.h>
18#include <linux/slab.h>
19#include <linux/workqueue.h>
Inki Daed87342c2012-11-03 21:53:24 -070020#include <linux/dma-mapping.h>
21#include <linux/dma-attrs.h>
Ajay Kumar95fc6332013-02-06 10:59:44 +053022#include <linux/of.h>
Joonyoung Shimd7f16422012-05-17 20:06:32 +090023
David Howells760285e2012-10-02 18:01:07 +010024#include <drm/drmP.h>
25#include <drm/exynos_drm.h>
Joonyoung Shimd7f16422012-05-17 20:06:32 +090026#include "exynos_drm_drv.h"
27#include "exynos_drm_gem.h"
Inki Daed87342c2012-11-03 21:53:24 -070028#include "exynos_drm_iommu.h"
Joonyoung Shimd7f16422012-05-17 20:06:32 +090029
30#define G2D_HW_MAJOR_VER 4
31#define G2D_HW_MINOR_VER 1
32
33/* vaild register range set from user: 0x0104 ~ 0x0880 */
34#define G2D_VALID_START 0x0104
35#define G2D_VALID_END 0x0880
36
37/* general registers */
38#define G2D_SOFT_RESET 0x0000
39#define G2D_INTEN 0x0004
40#define G2D_INTC_PEND 0x000C
41#define G2D_DMA_SFR_BASE_ADDR 0x0080
42#define G2D_DMA_COMMAND 0x0084
43#define G2D_DMA_STATUS 0x008C
44#define G2D_DMA_HOLD_CMD 0x0090
45
46/* command registers */
47#define G2D_BITBLT_START 0x0100
48
49/* registers for base address */
50#define G2D_SRC_BASE_ADDR 0x0304
51#define G2D_SRC_PLANE2_BASE_ADDR 0x0318
52#define G2D_DST_BASE_ADDR 0x0404
53#define G2D_DST_PLANE2_BASE_ADDR 0x0418
54#define G2D_PAT_BASE_ADDR 0x0500
55#define G2D_MSK_BASE_ADDR 0x0520
56
57/* G2D_SOFT_RESET */
58#define G2D_SFRCLEAR (1 << 1)
59#define G2D_R (1 << 0)
60
61/* G2D_INTEN */
62#define G2D_INTEN_ACF (1 << 3)
63#define G2D_INTEN_UCF (1 << 2)
64#define G2D_INTEN_GCF (1 << 1)
65#define G2D_INTEN_SCF (1 << 0)
66
67/* G2D_INTC_PEND */
68#define G2D_INTP_ACMD_FIN (1 << 3)
69#define G2D_INTP_UCMD_FIN (1 << 2)
70#define G2D_INTP_GCMD_FIN (1 << 1)
71#define G2D_INTP_SCMD_FIN (1 << 0)
72
73/* G2D_DMA_COMMAND */
74#define G2D_DMA_HALT (1 << 2)
75#define G2D_DMA_CONTINUE (1 << 1)
76#define G2D_DMA_START (1 << 0)
77
78/* G2D_DMA_STATUS */
79#define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)
80#define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)
81#define G2D_DMA_DONE (1 << 0)
82#define G2D_DMA_LIST_DONE_COUNT_OFFSET 17
83
84/* G2D_DMA_HOLD_CMD */
YoungJun Cho7ad01812013-03-13 16:44:37 +090085#define G2D_USER_HOLD (1 << 2)
Joonyoung Shimd7f16422012-05-17 20:06:32 +090086#define G2D_LIST_HOLD (1 << 1)
87#define G2D_BITBLT_HOLD (1 << 0)
88
89/* G2D_BITBLT_START */
90#define G2D_START_CASESEL (1 << 2)
91#define G2D_START_NHOLT (1 << 1)
92#define G2D_START_BITBLT (1 << 0)
93
94#define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)
95#define G2D_CMDLIST_NUM 64
96#define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
97#define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
98
Inki Dae2a3098f2012-11-04 05:48:52 -080099/* maximum buffer pool size of userptr is 64MB as default */
100#define MAX_POOL (64 * 1024 * 1024)
101
102enum {
103 BUF_TYPE_GEM = 1,
104 BUF_TYPE_USERPTR,
105};
106
YoungJun Cho9963cb62013-03-13 17:10:08 +0900107enum g2d_reg_type {
108 REG_TYPE_NONE = -1,
109 REG_TYPE_SRC,
110 REG_TYPE_SRC_PLANE2,
111 REG_TYPE_DST,
112 REG_TYPE_DST_PLANE2,
113 REG_TYPE_PAT,
114 REG_TYPE_MSK,
115 MAX_REG_TYPE_NR
116};
117
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900118/* cmdlist data structure */
119struct g2d_cmdlist {
Inki Dae2a3098f2012-11-04 05:48:52 -0800120 u32 head;
121 unsigned long data[G2D_CMDLIST_DATA_NUM];
122 u32 last; /* last data offset */
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900123};
124
YoungJun Cho9963cb62013-03-13 17:10:08 +0900125/*
126 * A structure of buffer information
127 *
128 * @map_nr: manages the number of mapped buffers
129 * @reg_types: stores regitster type in the order of requested command
130 * @handles: stores buffer handle in its reg_type position
131 * @types: stores buffer type in its reg_type position
132 *
133 */
134struct g2d_buf_info {
135 unsigned int map_nr;
136 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR];
137 unsigned long handles[MAX_REG_TYPE_NR];
138 unsigned int types[MAX_REG_TYPE_NR];
139};
140
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900141struct drm_exynos_pending_g2d_event {
142 struct drm_pending_event base;
143 struct drm_exynos_g2d_event event;
144};
145
Inki Dae2a3098f2012-11-04 05:48:52 -0800146struct g2d_cmdlist_userptr {
147 struct list_head list;
148 dma_addr_t dma_addr;
149 unsigned long userptr;
150 unsigned long size;
151 struct page **pages;
152 unsigned int npages;
153 struct sg_table *sgt;
154 struct vm_area_struct *vma;
155 atomic_t refcount;
156 bool in_pool;
157 bool out_of_list;
158};
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900159struct g2d_cmdlist_node {
160 struct list_head list;
161 struct g2d_cmdlist *cmdlist;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900162 dma_addr_t dma_addr;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900163 struct g2d_buf_info buf_info;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900164
165 struct drm_exynos_pending_g2d_event *event;
166};
167
168struct g2d_runqueue_node {
169 struct list_head list;
170 struct list_head run_cmdlist;
171 struct list_head event_list;
Inki Daed87342c2012-11-03 21:53:24 -0700172 struct drm_file *filp;
Inki Dae6b6bae22012-09-11 10:45:36 +0900173 pid_t pid;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900174 struct completion complete;
175 int async;
176};
177
178struct g2d_data {
179 struct device *dev;
180 struct clk *gate_clk;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900181 void __iomem *regs;
182 int irq;
183 struct workqueue_struct *g2d_workq;
184 struct work_struct runqueue_work;
185 struct exynos_drm_subdrv subdrv;
186 bool suspended;
187
188 /* cmdlist */
189 struct g2d_cmdlist_node *cmdlist_node;
190 struct list_head free_cmdlist;
191 struct mutex cmdlist_mutex;
192 dma_addr_t cmdlist_pool;
193 void *cmdlist_pool_virt;
Inki Daed87342c2012-11-03 21:53:24 -0700194 struct dma_attrs cmdlist_dma_attrs;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900195
196 /* runqueue*/
197 struct g2d_runqueue_node *runqueue_node;
198 struct list_head runqueue;
199 struct mutex runqueue_mutex;
200 struct kmem_cache *runqueue_slab;
Inki Dae2a3098f2012-11-04 05:48:52 -0800201
202 unsigned long current_pool;
203 unsigned long max_pool;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900204};
205
206static int g2d_init_cmdlist(struct g2d_data *g2d)
207{
208 struct device *dev = g2d->dev;
209 struct g2d_cmdlist_node *node = g2d->cmdlist_node;
Inki Daed87342c2012-11-03 21:53:24 -0700210 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900211 int nr;
212 int ret;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900213 struct g2d_buf_info *buf_info;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900214
Inki Daed87342c2012-11-03 21:53:24 -0700215 init_dma_attrs(&g2d->cmdlist_dma_attrs);
216 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &g2d->cmdlist_dma_attrs);
217
218 g2d->cmdlist_pool_virt = dma_alloc_attrs(subdrv->drm_dev->dev,
219 G2D_CMDLIST_POOL_SIZE,
220 &g2d->cmdlist_pool, GFP_KERNEL,
221 &g2d->cmdlist_dma_attrs);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900222 if (!g2d->cmdlist_pool_virt) {
223 dev_err(dev, "failed to allocate dma memory\n");
224 return -ENOMEM;
225 }
226
Joonyoung Shimfab9f8d2012-09-27 19:26:03 +0900227 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900228 if (!node) {
229 dev_err(dev, "failed to allocate memory\n");
230 ret = -ENOMEM;
231 goto err;
232 }
233
234 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
YoungJun Cho9963cb62013-03-13 17:10:08 +0900235 unsigned int i;
236
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900237 node[nr].cmdlist =
238 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
239 node[nr].dma_addr =
240 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
241
YoungJun Cho9963cb62013-03-13 17:10:08 +0900242 buf_info = &node[nr].buf_info;
243 for (i = 0; i < MAX_REG_TYPE_NR; i++)
244 buf_info->reg_types[i] = REG_TYPE_NONE;
245
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900246 list_add_tail(&node[nr].list, &g2d->free_cmdlist);
247 }
248
249 return 0;
250
251err:
Inki Daed87342c2012-11-03 21:53:24 -0700252 dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
253 g2d->cmdlist_pool_virt,
254 g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900255 return ret;
256}
257
258static void g2d_fini_cmdlist(struct g2d_data *g2d)
259{
Inki Daed87342c2012-11-03 21:53:24 -0700260 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900261
262 kfree(g2d->cmdlist_node);
Inki Daed87342c2012-11-03 21:53:24 -0700263 dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
264 g2d->cmdlist_pool_virt,
265 g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900266}
267
268static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
269{
270 struct device *dev = g2d->dev;
271 struct g2d_cmdlist_node *node;
272
273 mutex_lock(&g2d->cmdlist_mutex);
274 if (list_empty(&g2d->free_cmdlist)) {
275 dev_err(dev, "there is no free cmdlist\n");
276 mutex_unlock(&g2d->cmdlist_mutex);
277 return NULL;
278 }
279
280 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
281 list);
282 list_del_init(&node->list);
283 mutex_unlock(&g2d->cmdlist_mutex);
284
285 return node;
286}
287
288static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
289{
290 mutex_lock(&g2d->cmdlist_mutex);
291 list_move_tail(&node->list, &g2d->free_cmdlist);
292 mutex_unlock(&g2d->cmdlist_mutex);
293}
294
295static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv,
296 struct g2d_cmdlist_node *node)
297{
298 struct g2d_cmdlist_node *lnode;
299
300 if (list_empty(&g2d_priv->inuse_cmdlist))
301 goto add_to_list;
302
303 /* this links to base address of new cmdlist */
304 lnode = list_entry(g2d_priv->inuse_cmdlist.prev,
305 struct g2d_cmdlist_node, list);
306 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
307
308add_to_list:
309 list_add_tail(&node->list, &g2d_priv->inuse_cmdlist);
310
311 if (node->event)
312 list_add_tail(&node->event->base.link, &g2d_priv->event_list);
313}
314
Inki Dae2a3098f2012-11-04 05:48:52 -0800315static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev,
316 unsigned long obj,
317 bool force)
318{
319 struct g2d_cmdlist_userptr *g2d_userptr =
320 (struct g2d_cmdlist_userptr *)obj;
321
322 if (!obj)
323 return;
324
325 if (force)
326 goto out;
327
328 atomic_dec(&g2d_userptr->refcount);
329
330 if (atomic_read(&g2d_userptr->refcount) > 0)
331 return;
332
333 if (g2d_userptr->in_pool)
334 return;
335
336out:
337 exynos_gem_unmap_sgt_from_dma(drm_dev, g2d_userptr->sgt,
338 DMA_BIDIRECTIONAL);
339
340 exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
341 g2d_userptr->npages,
342 g2d_userptr->vma);
343
344 if (!g2d_userptr->out_of_list)
345 list_del_init(&g2d_userptr->list);
346
347 sg_free_table(g2d_userptr->sgt);
348 kfree(g2d_userptr->sgt);
349 g2d_userptr->sgt = NULL;
350
351 kfree(g2d_userptr->pages);
Inki Dae2a3098f2012-11-04 05:48:52 -0800352 g2d_userptr->pages = NULL;
Sachin Kamatdf3d90e2012-11-23 09:11:59 +0530353 kfree(g2d_userptr);
Inki Dae2a3098f2012-11-04 05:48:52 -0800354 g2d_userptr = NULL;
355}
356
Sachin Kamatb7848c72013-01-14 12:29:09 +0530357static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
Inki Dae2a3098f2012-11-04 05:48:52 -0800358 unsigned long userptr,
359 unsigned long size,
360 struct drm_file *filp,
361 unsigned long *obj)
362{
363 struct drm_exynos_file_private *file_priv = filp->driver_priv;
364 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
365 struct g2d_cmdlist_userptr *g2d_userptr;
366 struct g2d_data *g2d;
367 struct page **pages;
368 struct sg_table *sgt;
369 struct vm_area_struct *vma;
370 unsigned long start, end;
371 unsigned int npages, offset;
372 int ret;
373
374 if (!size) {
375 DRM_ERROR("invalid userptr size.\n");
376 return ERR_PTR(-EINVAL);
377 }
378
379 g2d = dev_get_drvdata(g2d_priv->dev);
380
381 /* check if userptr already exists in userptr_list. */
382 list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) {
383 if (g2d_userptr->userptr == userptr) {
384 /*
385 * also check size because there could be same address
386 * and different size.
387 */
388 if (g2d_userptr->size == size) {
389 atomic_inc(&g2d_userptr->refcount);
390 *obj = (unsigned long)g2d_userptr;
391
392 return &g2d_userptr->dma_addr;
393 }
394
395 /*
396 * at this moment, maybe g2d dma is accessing this
397 * g2d_userptr memory region so just remove this
398 * g2d_userptr object from userptr_list not to be
399 * referred again and also except it the userptr
400 * pool to be released after the dma access completion.
401 */
402 g2d_userptr->out_of_list = true;
403 g2d_userptr->in_pool = false;
404 list_del_init(&g2d_userptr->list);
405
406 break;
407 }
408 }
409
410 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
411 if (!g2d_userptr) {
412 DRM_ERROR("failed to allocate g2d_userptr.\n");
413 return ERR_PTR(-ENOMEM);
414 }
415
416 atomic_set(&g2d_userptr->refcount, 1);
417
418 start = userptr & PAGE_MASK;
419 offset = userptr & ~PAGE_MASK;
420 end = PAGE_ALIGN(userptr + size);
421 npages = (end - start) >> PAGE_SHIFT;
422 g2d_userptr->npages = npages;
423
424 pages = kzalloc(npages * sizeof(struct page *), GFP_KERNEL);
425 if (!pages) {
426 DRM_ERROR("failed to allocate pages.\n");
427 kfree(g2d_userptr);
428 return ERR_PTR(-ENOMEM);
429 }
430
431 vma = find_vma(current->mm, userptr);
432 if (!vma) {
433 DRM_ERROR("failed to get vm region.\n");
434 ret = -EFAULT;
435 goto err_free_pages;
436 }
437
438 if (vma->vm_end < userptr + size) {
439 DRM_ERROR("vma is too small.\n");
440 ret = -EFAULT;
441 goto err_free_pages;
442 }
443
444 g2d_userptr->vma = exynos_gem_get_vma(vma);
445 if (!g2d_userptr->vma) {
446 DRM_ERROR("failed to copy vma.\n");
447 ret = -ENOMEM;
448 goto err_free_pages;
449 }
450
451 g2d_userptr->size = size;
452
453 ret = exynos_gem_get_pages_from_userptr(start & PAGE_MASK,
454 npages, pages, vma);
455 if (ret < 0) {
456 DRM_ERROR("failed to get user pages from userptr.\n");
457 goto err_put_vma;
458 }
459
460 g2d_userptr->pages = pages;
461
Sachin Kamate44a5c02013-01-25 14:45:42 +0530462 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
Inki Dae2a3098f2012-11-04 05:48:52 -0800463 if (!sgt) {
464 DRM_ERROR("failed to allocate sg table.\n");
465 ret = -ENOMEM;
466 goto err_free_userptr;
467 }
468
469 ret = sg_alloc_table_from_pages(sgt, pages, npages, offset,
470 size, GFP_KERNEL);
471 if (ret < 0) {
472 DRM_ERROR("failed to get sgt from pages.\n");
473 goto err_free_sgt;
474 }
475
476 g2d_userptr->sgt = sgt;
477
478 ret = exynos_gem_map_sgt_with_dma(drm_dev, g2d_userptr->sgt,
479 DMA_BIDIRECTIONAL);
480 if (ret < 0) {
481 DRM_ERROR("failed to map sgt with dma region.\n");
YoungJun Cho067ed332013-03-11 19:48:05 +0900482 goto err_sg_free_table;
Inki Dae2a3098f2012-11-04 05:48:52 -0800483 }
484
485 g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
486 g2d_userptr->userptr = userptr;
487
488 list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list);
489
490 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
491 g2d->current_pool += npages << PAGE_SHIFT;
492 g2d_userptr->in_pool = true;
493 }
494
495 *obj = (unsigned long)g2d_userptr;
496
497 return &g2d_userptr->dma_addr;
498
YoungJun Cho067ed332013-03-11 19:48:05 +0900499err_sg_free_table:
Inki Dae2a3098f2012-11-04 05:48:52 -0800500 sg_free_table(sgt);
YoungJun Cho067ed332013-03-11 19:48:05 +0900501
502err_free_sgt:
Inki Dae2a3098f2012-11-04 05:48:52 -0800503 kfree(sgt);
504 sgt = NULL;
505
506err_free_userptr:
507 exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
508 g2d_userptr->npages,
509 g2d_userptr->vma);
510
511err_put_vma:
512 exynos_gem_put_vma(g2d_userptr->vma);
513
514err_free_pages:
515 kfree(pages);
516 kfree(g2d_userptr);
517 pages = NULL;
518 g2d_userptr = NULL;
519
520 return ERR_PTR(ret);
521}
522
523static void g2d_userptr_free_all(struct drm_device *drm_dev,
524 struct g2d_data *g2d,
525 struct drm_file *filp)
526{
527 struct drm_exynos_file_private *file_priv = filp->driver_priv;
528 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
529 struct g2d_cmdlist_userptr *g2d_userptr, *n;
530
531 list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list)
532 if (g2d_userptr->in_pool)
533 g2d_userptr_put_dma_addr(drm_dev,
534 (unsigned long)g2d_userptr,
535 true);
536
537 g2d->current_pool = 0;
538}
539
YoungJun Cho9963cb62013-03-13 17:10:08 +0900540static enum g2d_reg_type g2d_get_reg_type(int reg_offset)
541{
542 enum g2d_reg_type reg_type;
543
544 switch (reg_offset) {
545 case G2D_SRC_BASE_ADDR:
546 reg_type = REG_TYPE_SRC;
547 break;
548 case G2D_SRC_PLANE2_BASE_ADDR:
549 reg_type = REG_TYPE_SRC_PLANE2;
550 break;
551 case G2D_DST_BASE_ADDR:
552 reg_type = REG_TYPE_DST;
553 break;
554 case G2D_DST_PLANE2_BASE_ADDR:
555 reg_type = REG_TYPE_DST_PLANE2;
556 break;
557 case G2D_PAT_BASE_ADDR:
558 reg_type = REG_TYPE_PAT;
559 break;
560 case G2D_MSK_BASE_ADDR:
561 reg_type = REG_TYPE_MSK;
562 break;
563 default:
564 reg_type = REG_TYPE_NONE;
565 DRM_ERROR("Unknown register offset![%d]\n", reg_offset);
566 break;
567 };
568
569 return reg_type;
570}
571
Inki Daed87342c2012-11-03 21:53:24 -0700572static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
573 struct g2d_cmdlist_node *node,
574 struct drm_device *drm_dev,
575 struct drm_file *file)
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900576{
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900577 struct g2d_cmdlist *cmdlist = node->cmdlist;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900578 struct g2d_buf_info *buf_info = &node->buf_info;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900579 int offset;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900580 int ret;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900581 int i;
582
YoungJun Cho9963cb62013-03-13 17:10:08 +0900583 for (i = 0; i < buf_info->map_nr; i++) {
584 enum g2d_reg_type reg_type;
585 int reg_pos;
Inki Daed87342c2012-11-03 21:53:24 -0700586 unsigned long handle;
587 dma_addr_t *addr;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900588
YoungJun Cho9963cb62013-03-13 17:10:08 +0900589 reg_pos = cmdlist->last - 2 * (i + 1);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900590
YoungJun Cho9963cb62013-03-13 17:10:08 +0900591 offset = cmdlist->data[reg_pos];
592 handle = cmdlist->data[reg_pos + 1];
593
594 reg_type = g2d_get_reg_type(offset);
595 if (reg_type == REG_TYPE_NONE) {
596 ret = -EFAULT;
597 goto err;
598 }
599
600 if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
Inki Dae2a3098f2012-11-04 05:48:52 -0800601 addr = exynos_drm_gem_get_dma_addr(drm_dev, handle,
602 file);
603 if (IS_ERR(addr)) {
YoungJun Cho9963cb62013-03-13 17:10:08 +0900604 ret = -EFAULT;
605 goto err;
Inki Dae2a3098f2012-11-04 05:48:52 -0800606 }
607 } else {
608 struct drm_exynos_g2d_userptr g2d_userptr;
609
610 if (copy_from_user(&g2d_userptr, (void __user *)handle,
611 sizeof(struct drm_exynos_g2d_userptr))) {
YoungJun Cho9963cb62013-03-13 17:10:08 +0900612 ret = -EFAULT;
613 goto err;
Inki Dae2a3098f2012-11-04 05:48:52 -0800614 }
615
616 addr = g2d_userptr_get_dma_addr(drm_dev,
617 g2d_userptr.userptr,
618 g2d_userptr.size,
619 file,
620 &handle);
621 if (IS_ERR(addr)) {
YoungJun Cho9963cb62013-03-13 17:10:08 +0900622 ret = -EFAULT;
623 goto err;
Inki Dae2a3098f2012-11-04 05:48:52 -0800624 }
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900625 }
626
YoungJun Cho9963cb62013-03-13 17:10:08 +0900627 cmdlist->data[reg_pos + 1] = *addr;
628 buf_info->reg_types[i] = reg_type;
629 buf_info->handles[reg_type] = handle;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900630 }
631
632 return 0;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900633
634err:
635 buf_info->map_nr = i;
636 return ret;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900637}
638
Inki Daed87342c2012-11-03 21:53:24 -0700639static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
640 struct g2d_cmdlist_node *node,
641 struct drm_file *filp)
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900642{
Inki Daed87342c2012-11-03 21:53:24 -0700643 struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
YoungJun Cho9963cb62013-03-13 17:10:08 +0900644 struct g2d_buf_info *buf_info = &node->buf_info;
Inki Daed87342c2012-11-03 21:53:24 -0700645 int i;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900646
YoungJun Cho9963cb62013-03-13 17:10:08 +0900647 for (i = 0; i < buf_info->map_nr; i++) {
648 enum g2d_reg_type reg_type;
649 unsigned long handle;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900650
YoungJun Cho9963cb62013-03-13 17:10:08 +0900651 reg_type = buf_info->reg_types[i];
652
653 handle = buf_info->handles[reg_type];
654
655 if (buf_info->types[reg_type] == BUF_TYPE_GEM)
Inki Dae2a3098f2012-11-04 05:48:52 -0800656 exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle,
657 filp);
658 else
659 g2d_userptr_put_dma_addr(subdrv->drm_dev, handle,
660 false);
Inki Daed87342c2012-11-03 21:53:24 -0700661
YoungJun Cho9963cb62013-03-13 17:10:08 +0900662 buf_info->reg_types[i] = REG_TYPE_NONE;
663 buf_info->handles[reg_type] = 0;
664 buf_info->types[reg_type] = 0;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900665 }
Inki Daed87342c2012-11-03 21:53:24 -0700666
YoungJun Cho9963cb62013-03-13 17:10:08 +0900667 buf_info->map_nr = 0;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900668}
669
670static void g2d_dma_start(struct g2d_data *g2d,
671 struct g2d_runqueue_node *runqueue_node)
672{
673 struct g2d_cmdlist_node *node =
674 list_first_entry(&runqueue_node->run_cmdlist,
675 struct g2d_cmdlist_node, list);
676
677 pm_runtime_get_sync(g2d->dev);
678 clk_enable(g2d->gate_clk);
679
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900680 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
681 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
682}
683
684static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
685{
686 struct g2d_runqueue_node *runqueue_node;
687
688 if (list_empty(&g2d->runqueue))
689 return NULL;
690
691 runqueue_node = list_first_entry(&g2d->runqueue,
692 struct g2d_runqueue_node, list);
693 list_del_init(&runqueue_node->list);
694 return runqueue_node;
695}
696
697static void g2d_free_runqueue_node(struct g2d_data *g2d,
698 struct g2d_runqueue_node *runqueue_node)
699{
Inki Daed87342c2012-11-03 21:53:24 -0700700 struct g2d_cmdlist_node *node;
701
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900702 if (!runqueue_node)
703 return;
704
705 mutex_lock(&g2d->cmdlist_mutex);
Inki Daed87342c2012-11-03 21:53:24 -0700706 /*
707 * commands in run_cmdlist have been completed so unmap all gem
708 * objects in each command node so that they are unreferenced.
709 */
710 list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
711 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900712 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
713 mutex_unlock(&g2d->cmdlist_mutex);
714
715 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
716}
717
718static void g2d_exec_runqueue(struct g2d_data *g2d)
719{
720 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
721 if (g2d->runqueue_node)
722 g2d_dma_start(g2d, g2d->runqueue_node);
723}
724
725static void g2d_runqueue_worker(struct work_struct *work)
726{
727 struct g2d_data *g2d = container_of(work, struct g2d_data,
728 runqueue_work);
729
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900730 mutex_lock(&g2d->runqueue_mutex);
731 clk_disable(g2d->gate_clk);
732 pm_runtime_put_sync(g2d->dev);
733
734 complete(&g2d->runqueue_node->complete);
735 if (g2d->runqueue_node->async)
736 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
737
738 if (g2d->suspended)
739 g2d->runqueue_node = NULL;
740 else
741 g2d_exec_runqueue(g2d);
742 mutex_unlock(&g2d->runqueue_mutex);
743}
744
745static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
746{
747 struct drm_device *drm_dev = g2d->subdrv.drm_dev;
748 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
749 struct drm_exynos_pending_g2d_event *e;
750 struct timeval now;
751 unsigned long flags;
752
753 if (list_empty(&runqueue_node->event_list))
754 return;
755
756 e = list_first_entry(&runqueue_node->event_list,
757 struct drm_exynos_pending_g2d_event, base.link);
758
759 do_gettimeofday(&now);
760 e->event.tv_sec = now.tv_sec;
761 e->event.tv_usec = now.tv_usec;
762 e->event.cmdlist_no = cmdlist_no;
763
764 spin_lock_irqsave(&drm_dev->event_lock, flags);
765 list_move_tail(&e->base.link, &e->base.file_priv->event_list);
766 wake_up_interruptible(&e->base.file_priv->event_wait);
767 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
768}
769
770static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
771{
772 struct g2d_data *g2d = dev_id;
773 u32 pending;
774
775 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
776 if (pending)
777 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
778
779 if (pending & G2D_INTP_GCMD_FIN) {
780 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
781
782 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
783 G2D_DMA_LIST_DONE_COUNT_OFFSET;
784
785 g2d_finish_event(g2d, cmdlist_no);
786
787 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
788 if (!(pending & G2D_INTP_ACMD_FIN)) {
789 writel_relaxed(G2D_DMA_CONTINUE,
790 g2d->regs + G2D_DMA_COMMAND);
791 }
792 }
793
794 if (pending & G2D_INTP_ACMD_FIN)
795 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
796
797 return IRQ_HANDLED;
798}
799
Inki Dae2a3098f2012-11-04 05:48:52 -0800800static int g2d_check_reg_offset(struct device *dev,
801 struct g2d_cmdlist_node *node,
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900802 int nr, bool for_addr)
803{
Inki Dae2a3098f2012-11-04 05:48:52 -0800804 struct g2d_cmdlist *cmdlist = node->cmdlist;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900805 int reg_offset;
806 int index;
807 int i;
808
809 for (i = 0; i < nr; i++) {
YoungJun Cho9963cb62013-03-13 17:10:08 +0900810 struct g2d_buf_info *buf_info = &node->buf_info;
811 enum g2d_reg_type reg_type;
812
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900813 index = cmdlist->last - 2 * (i + 1);
Inki Dae2a3098f2012-11-04 05:48:52 -0800814
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900815 reg_offset = cmdlist->data[index] & ~0xfffff000;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900816 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
817 goto err;
818 if (reg_offset % 4)
819 goto err;
820
821 switch (reg_offset) {
822 case G2D_SRC_BASE_ADDR:
823 case G2D_SRC_PLANE2_BASE_ADDR:
824 case G2D_DST_BASE_ADDR:
825 case G2D_DST_PLANE2_BASE_ADDR:
826 case G2D_PAT_BASE_ADDR:
827 case G2D_MSK_BASE_ADDR:
828 if (!for_addr)
829 goto err;
Inki Dae2a3098f2012-11-04 05:48:52 -0800830
YoungJun Cho9963cb62013-03-13 17:10:08 +0900831 reg_type = g2d_get_reg_type(reg_offset);
832 if (reg_type == REG_TYPE_NONE)
833 goto err;
834
835 /* check userptr buffer type. */
836 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
837 buf_info->types[reg_type] = BUF_TYPE_USERPTR;
838 cmdlist->data[index] &= ~G2D_BUF_USERPTR;
839 } else
840 buf_info->types[reg_type] = BUF_TYPE_GEM;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900841 break;
842 default:
843 if (for_addr)
844 goto err;
845 break;
846 }
847 }
848
849 return 0;
850
851err:
Inki Dae2a3098f2012-11-04 05:48:52 -0800852 dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900853 return -EINVAL;
854}
855
856/* ioctl functions */
857int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
858 struct drm_file *file)
859{
860 struct drm_exynos_g2d_get_ver *ver = data;
861
862 ver->major = G2D_HW_MAJOR_VER;
863 ver->minor = G2D_HW_MINOR_VER;
864
865 return 0;
866}
867EXPORT_SYMBOL_GPL(exynos_g2d_get_ver_ioctl);
868
869int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
870 struct drm_file *file)
871{
872 struct drm_exynos_file_private *file_priv = file->driver_priv;
873 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
874 struct device *dev = g2d_priv->dev;
875 struct g2d_data *g2d;
876 struct drm_exynos_g2d_set_cmdlist *req = data;
877 struct drm_exynos_g2d_cmd *cmd;
878 struct drm_exynos_pending_g2d_event *e;
879 struct g2d_cmdlist_node *node;
880 struct g2d_cmdlist *cmdlist;
881 unsigned long flags;
882 int size;
883 int ret;
884
885 if (!dev)
886 return -ENODEV;
887
888 g2d = dev_get_drvdata(dev);
889 if (!g2d)
890 return -EFAULT;
891
892 node = g2d_get_cmdlist(g2d);
893 if (!node)
894 return -ENOMEM;
895
896 node->event = NULL;
897
898 if (req->event_type != G2D_EVENT_NOT) {
899 spin_lock_irqsave(&drm_dev->event_lock, flags);
900 if (file->event_space < sizeof(e->event)) {
901 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
902 ret = -ENOMEM;
903 goto err;
904 }
905 file->event_space -= sizeof(e->event);
906 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
907
908 e = kzalloc(sizeof(*node->event), GFP_KERNEL);
909 if (!e) {
910 dev_err(dev, "failed to allocate event\n");
911
912 spin_lock_irqsave(&drm_dev->event_lock, flags);
913 file->event_space += sizeof(e->event);
914 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
915
916 ret = -ENOMEM;
917 goto err;
918 }
919
920 e->event.base.type = DRM_EXYNOS_G2D_EVENT;
921 e->event.base.length = sizeof(e->event);
922 e->event.user_data = req->user_data;
923 e->base.event = &e->event.base;
924 e->base.file_priv = file;
925 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
926
927 node->event = e;
928 }
929
930 cmdlist = node->cmdlist;
931
932 cmdlist->last = 0;
933
934 /*
935 * If don't clear SFR registers, the cmdlist is affected by register
936 * values of previous cmdlist. G2D hw executes SFR clear command and
937 * a next command at the same time then the next command is ignored and
938 * is executed rightly from next next command, so needs a dummy command
939 * to next command of SFR clear command.
940 */
941 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
942 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
943 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
944 cmdlist->data[cmdlist->last++] = 0;
945
YoungJun Cho7ad01812013-03-13 16:44:37 +0900946 /*
947 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
948 * and GCF bit should be set to INTEN register if user wants
949 * G2D interrupt event once current command list execution is
950 * finished.
951 * Otherwise only ACF bit should be set to INTEN register so
952 * that one interrupt is occured after all command lists
953 * have been completed.
954 */
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900955 if (node->event) {
YoungJun Cho7ad01812013-03-13 16:44:37 +0900956 cmdlist->data[cmdlist->last++] = G2D_INTEN;
957 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900958 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
959 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
YoungJun Cho7ad01812013-03-13 16:44:37 +0900960 } else {
961 cmdlist->data[cmdlist->last++] = G2D_INTEN;
962 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900963 }
964
965 /* Check size of cmdlist: last 2 is about G2D_BITBLT_START */
Inki Dae2a3098f2012-11-04 05:48:52 -0800966 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900967 if (size > G2D_CMDLIST_DATA_NUM) {
968 dev_err(dev, "cmdlist size is too big\n");
969 ret = -EINVAL;
970 goto err_free_event;
971 }
972
973 cmd = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd;
974
975 if (copy_from_user(cmdlist->data + cmdlist->last,
976 (void __user *)cmd,
977 sizeof(*cmd) * req->cmd_nr)) {
978 ret = -EFAULT;
979 goto err_free_event;
980 }
981 cmdlist->last += req->cmd_nr * 2;
982
Inki Dae2a3098f2012-11-04 05:48:52 -0800983 ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false);
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900984 if (ret < 0)
985 goto err_free_event;
986
YoungJun Cho9963cb62013-03-13 17:10:08 +0900987 node->buf_info.map_nr = req->cmd_buf_nr;
Inki Dae2a3098f2012-11-04 05:48:52 -0800988 if (req->cmd_buf_nr) {
989 struct drm_exynos_g2d_cmd *cmd_buf;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900990
Inki Dae2a3098f2012-11-04 05:48:52 -0800991 cmd_buf = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd_buf;
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900992
993 if (copy_from_user(cmdlist->data + cmdlist->last,
Inki Dae2a3098f2012-11-04 05:48:52 -0800994 (void __user *)cmd_buf,
995 sizeof(*cmd_buf) * req->cmd_buf_nr)) {
Joonyoung Shimd7f16422012-05-17 20:06:32 +0900996 ret = -EFAULT;
997 goto err_free_event;
998 }
Inki Dae2a3098f2012-11-04 05:48:52 -0800999 cmdlist->last += req->cmd_buf_nr * 2;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001000
Inki Dae2a3098f2012-11-04 05:48:52 -08001001 ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001002 if (ret < 0)
1003 goto err_free_event;
1004
Inki Daed87342c2012-11-03 21:53:24 -07001005 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001006 if (ret < 0)
1007 goto err_unmap;
1008 }
1009
1010 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1011 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1012
1013 /* head */
1014 cmdlist->head = cmdlist->last / 2;
1015
1016 /* tail */
1017 cmdlist->data[cmdlist->last] = 0;
1018
1019 g2d_add_cmdlist_to_inuse(g2d_priv, node);
1020
1021 return 0;
1022
1023err_unmap:
Inki Daed87342c2012-11-03 21:53:24 -07001024 g2d_unmap_cmdlist_gem(g2d, node, file);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001025err_free_event:
1026 if (node->event) {
1027 spin_lock_irqsave(&drm_dev->event_lock, flags);
1028 file->event_space += sizeof(e->event);
1029 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
1030 kfree(node->event);
1031 }
1032err:
1033 g2d_put_cmdlist(g2d, node);
1034 return ret;
1035}
1036EXPORT_SYMBOL_GPL(exynos_g2d_set_cmdlist_ioctl);
1037
1038int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1039 struct drm_file *file)
1040{
1041 struct drm_exynos_file_private *file_priv = file->driver_priv;
1042 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
1043 struct device *dev = g2d_priv->dev;
1044 struct g2d_data *g2d;
1045 struct drm_exynos_g2d_exec *req = data;
1046 struct g2d_runqueue_node *runqueue_node;
1047 struct list_head *run_cmdlist;
1048 struct list_head *event_list;
1049
1050 if (!dev)
1051 return -ENODEV;
1052
1053 g2d = dev_get_drvdata(dev);
1054 if (!g2d)
1055 return -EFAULT;
1056
1057 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
1058 if (!runqueue_node) {
1059 dev_err(dev, "failed to allocate memory\n");
1060 return -ENOMEM;
1061 }
1062 run_cmdlist = &runqueue_node->run_cmdlist;
1063 event_list = &runqueue_node->event_list;
1064 INIT_LIST_HEAD(run_cmdlist);
1065 INIT_LIST_HEAD(event_list);
1066 init_completion(&runqueue_node->complete);
1067 runqueue_node->async = req->async;
1068
1069 list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist);
1070 list_splice_init(&g2d_priv->event_list, event_list);
1071
1072 if (list_empty(run_cmdlist)) {
1073 dev_err(dev, "there is no inuse cmdlist\n");
1074 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
1075 return -EPERM;
1076 }
1077
1078 mutex_lock(&g2d->runqueue_mutex);
Inki Dae6b6bae22012-09-11 10:45:36 +09001079 runqueue_node->pid = current->pid;
Inki Daed87342c2012-11-03 21:53:24 -07001080 runqueue_node->filp = file;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001081 list_add_tail(&runqueue_node->list, &g2d->runqueue);
1082 if (!g2d->runqueue_node)
1083 g2d_exec_runqueue(g2d);
1084 mutex_unlock(&g2d->runqueue_mutex);
1085
1086 if (runqueue_node->async)
1087 goto out;
1088
1089 wait_for_completion(&runqueue_node->complete);
1090 g2d_free_runqueue_node(g2d, runqueue_node);
1091
1092out:
1093 return 0;
1094}
1095EXPORT_SYMBOL_GPL(exynos_g2d_exec_ioctl);
1096
Inki Daed87342c2012-11-03 21:53:24 -07001097static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
1098{
1099 struct g2d_data *g2d;
1100 int ret;
1101
1102 g2d = dev_get_drvdata(dev);
1103 if (!g2d)
1104 return -EFAULT;
1105
1106 /* allocate dma-aware cmdlist buffer. */
1107 ret = g2d_init_cmdlist(g2d);
1108 if (ret < 0) {
1109 dev_err(dev, "cmdlist init failed\n");
1110 return ret;
1111 }
1112
1113 if (!is_drm_iommu_supported(drm_dev))
1114 return 0;
1115
1116 ret = drm_iommu_attach_device(drm_dev, dev);
1117 if (ret < 0) {
1118 dev_err(dev, "failed to enable iommu.\n");
1119 g2d_fini_cmdlist(g2d);
1120 }
1121
1122 return ret;
1123
1124}
1125
1126static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
1127{
1128 if (!is_drm_iommu_supported(drm_dev))
1129 return;
1130
1131 drm_iommu_detach_device(drm_dev, dev);
1132}
1133
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001134static int g2d_open(struct drm_device *drm_dev, struct device *dev,
1135 struct drm_file *file)
1136{
1137 struct drm_exynos_file_private *file_priv = file->driver_priv;
1138 struct exynos_drm_g2d_private *g2d_priv;
1139
1140 g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL);
1141 if (!g2d_priv) {
1142 dev_err(dev, "failed to allocate g2d private data\n");
1143 return -ENOMEM;
1144 }
1145
1146 g2d_priv->dev = dev;
1147 file_priv->g2d_priv = g2d_priv;
1148
1149 INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist);
1150 INIT_LIST_HEAD(&g2d_priv->event_list);
Inki Dae2a3098f2012-11-04 05:48:52 -08001151 INIT_LIST_HEAD(&g2d_priv->userptr_list);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001152
1153 return 0;
1154}
1155
1156static void g2d_close(struct drm_device *drm_dev, struct device *dev,
1157 struct drm_file *file)
1158{
1159 struct drm_exynos_file_private *file_priv = file->driver_priv;
1160 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
1161 struct g2d_data *g2d;
1162 struct g2d_cmdlist_node *node, *n;
1163
1164 if (!dev)
1165 return;
1166
1167 g2d = dev_get_drvdata(dev);
1168 if (!g2d)
1169 return;
1170
1171 mutex_lock(&g2d->cmdlist_mutex);
Inki Daed87342c2012-11-03 21:53:24 -07001172 list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) {
1173 /*
1174 * unmap all gem objects not completed.
1175 *
1176 * P.S. if current process was terminated forcely then
1177 * there may be some commands in inuse_cmdlist so unmap
1178 * them.
1179 */
1180 g2d_unmap_cmdlist_gem(g2d, node, file);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001181 list_move_tail(&node->list, &g2d->free_cmdlist);
Inki Daed87342c2012-11-03 21:53:24 -07001182 }
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001183 mutex_unlock(&g2d->cmdlist_mutex);
1184
Inki Dae2a3098f2012-11-04 05:48:52 -08001185 /* release all g2d_userptr in pool. */
1186 g2d_userptr_free_all(drm_dev, g2d, file);
1187
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001188 kfree(file_priv->g2d_priv);
1189}
1190
Greg Kroah-Hartman56550d92012-12-21 15:09:25 -08001191static int g2d_probe(struct platform_device *pdev)
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001192{
1193 struct device *dev = &pdev->dev;
1194 struct resource *res;
1195 struct g2d_data *g2d;
1196 struct exynos_drm_subdrv *subdrv;
1197 int ret;
1198
Sachin Kamatb7675932012-08-06 12:16:20 +05301199 g2d = devm_kzalloc(&pdev->dev, sizeof(*g2d), GFP_KERNEL);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001200 if (!g2d) {
1201 dev_err(dev, "failed to allocate driver data\n");
1202 return -ENOMEM;
1203 }
1204
1205 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1206 sizeof(struct g2d_runqueue_node), 0, 0, NULL);
Sachin Kamatb7675932012-08-06 12:16:20 +05301207 if (!g2d->runqueue_slab)
1208 return -ENOMEM;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001209
1210 g2d->dev = dev;
1211
1212 g2d->g2d_workq = create_singlethread_workqueue("g2d");
1213 if (!g2d->g2d_workq) {
1214 dev_err(dev, "failed to create workqueue\n");
1215 ret = -EINVAL;
1216 goto err_destroy_slab;
1217 }
1218
1219 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1220 INIT_LIST_HEAD(&g2d->free_cmdlist);
1221 INIT_LIST_HEAD(&g2d->runqueue);
1222
1223 mutex_init(&g2d->cmdlist_mutex);
1224 mutex_init(&g2d->runqueue_mutex);
1225
Sachin Kamatdc625532012-11-23 09:11:58 +05301226 g2d->gate_clk = devm_clk_get(dev, "fimg2d");
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001227 if (IS_ERR(g2d->gate_clk)) {
1228 dev_err(dev, "failed to get gate clock\n");
1229 ret = PTR_ERR(g2d->gate_clk);
Inki Daed87342c2012-11-03 21:53:24 -07001230 goto err_destroy_workqueue;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001231 }
1232
1233 pm_runtime_enable(dev);
1234
1235 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001236
Thierry Redingd4ed6022013-01-21 11:09:02 +01001237 g2d->regs = devm_ioremap_resource(&pdev->dev, res);
1238 if (IS_ERR(g2d->regs)) {
1239 ret = PTR_ERR(g2d->regs);
Sachin Kamatb7675932012-08-06 12:16:20 +05301240 goto err_put_clk;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001241 }
1242
1243 g2d->irq = platform_get_irq(pdev, 0);
1244 if (g2d->irq < 0) {
1245 dev_err(dev, "failed to get irq\n");
1246 ret = g2d->irq;
Sachin Kamatb7675932012-08-06 12:16:20 +05301247 goto err_put_clk;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001248 }
1249
Sachin Kamatb7675932012-08-06 12:16:20 +05301250 ret = devm_request_irq(&pdev->dev, g2d->irq, g2d_irq_handler, 0,
1251 "drm_g2d", g2d);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001252 if (ret < 0) {
1253 dev_err(dev, "irq request failed\n");
Sachin Kamatb7675932012-08-06 12:16:20 +05301254 goto err_put_clk;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001255 }
1256
Inki Dae2a3098f2012-11-04 05:48:52 -08001257 g2d->max_pool = MAX_POOL;
1258
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001259 platform_set_drvdata(pdev, g2d);
1260
1261 subdrv = &g2d->subdrv;
1262 subdrv->dev = dev;
Inki Daed87342c2012-11-03 21:53:24 -07001263 subdrv->probe = g2d_subdrv_probe;
1264 subdrv->remove = g2d_subdrv_remove;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001265 subdrv->open = g2d_open;
1266 subdrv->close = g2d_close;
1267
1268 ret = exynos_drm_subdrv_register(subdrv);
1269 if (ret < 0) {
1270 dev_err(dev, "failed to register drm g2d device\n");
Sachin Kamatb7675932012-08-06 12:16:20 +05301271 goto err_put_clk;
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001272 }
1273
1274 dev_info(dev, "The exynos g2d(ver %d.%d) successfully probed\n",
1275 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1276
1277 return 0;
1278
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001279err_put_clk:
1280 pm_runtime_disable(dev);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001281err_destroy_workqueue:
1282 destroy_workqueue(g2d->g2d_workq);
1283err_destroy_slab:
1284 kmem_cache_destroy(g2d->runqueue_slab);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001285 return ret;
1286}
1287
Greg Kroah-Hartman56550d92012-12-21 15:09:25 -08001288static int g2d_remove(struct platform_device *pdev)
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001289{
1290 struct g2d_data *g2d = platform_get_drvdata(pdev);
1291
1292 cancel_work_sync(&g2d->runqueue_work);
1293 exynos_drm_subdrv_unregister(&g2d->subdrv);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001294
1295 while (g2d->runqueue_node) {
1296 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
1297 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
1298 }
1299
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001300 pm_runtime_disable(&pdev->dev);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001301
1302 g2d_fini_cmdlist(g2d);
1303 destroy_workqueue(g2d->g2d_workq);
1304 kmem_cache_destroy(g2d->runqueue_slab);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001305
1306 return 0;
1307}
1308
1309#ifdef CONFIG_PM_SLEEP
1310static int g2d_suspend(struct device *dev)
1311{
1312 struct g2d_data *g2d = dev_get_drvdata(dev);
1313
1314 mutex_lock(&g2d->runqueue_mutex);
1315 g2d->suspended = true;
1316 mutex_unlock(&g2d->runqueue_mutex);
1317
1318 while (g2d->runqueue_node)
1319 /* FIXME: good range? */
1320 usleep_range(500, 1000);
1321
Tejun Heo43829732012-08-20 14:51:24 -07001322 flush_work(&g2d->runqueue_work);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001323
1324 return 0;
1325}
1326
1327static int g2d_resume(struct device *dev)
1328{
1329 struct g2d_data *g2d = dev_get_drvdata(dev);
1330
1331 g2d->suspended = false;
1332 g2d_exec_runqueue(g2d);
1333
1334 return 0;
1335}
1336#endif
1337
Sachin Kamat9e1355e2012-08-28 14:11:41 +05301338static SIMPLE_DEV_PM_OPS(g2d_pm_ops, g2d_suspend, g2d_resume);
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001339
Ajay Kumar95fc6332013-02-06 10:59:44 +05301340#ifdef CONFIG_OF
1341static const struct of_device_id exynos_g2d_match[] = {
1342 { .compatible = "samsung,exynos5250-g2d" },
1343 {},
1344};
1345MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1346#endif
1347
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001348struct platform_driver g2d_driver = {
1349 .probe = g2d_probe,
Greg Kroah-Hartman56550d92012-12-21 15:09:25 -08001350 .remove = g2d_remove,
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001351 .driver = {
1352 .name = "s5p-g2d",
1353 .owner = THIS_MODULE,
1354 .pm = &g2d_pm_ops,
Ajay Kumar95fc6332013-02-06 10:59:44 +05301355 .of_match_table = of_match_ptr(exynos_g2d_match),
Joonyoung Shimd7f16422012-05-17 20:06:32 +09001356 },
1357};