blob: f94fcd752815e37c1a0cfdf532e05c829330f788 [file] [log] [blame]
Sascha Hauerb9d47452012-06-27 15:30:18 +02001/*
2 * drm gem CMA (contiguous memory allocator) helper functions
3 *
4 * Copyright (C) 2012 Sascha Hauer, Pengutronix
5 *
6 * Based on Samsung Exynos code
7 *
8 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version 2
13 * of the License, or (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#include <linux/mm.h>
21#include <linux/slab.h>
22#include <linux/mutex.h>
23#include <linux/export.h>
Laurent Pinchart71d72822013-02-17 01:57:30 +010024#include <linux/dma-buf.h>
Sascha Hauerb9d47452012-06-27 15:30:18 +020025#include <linux/dma-mapping.h>
26
27#include <drm/drmP.h>
28#include <drm/drm.h>
29#include <drm/drm_gem_cma_helper.h>
30
31static unsigned int get_gem_mmap_offset(struct drm_gem_object *obj)
32{
33 return (unsigned int)obj->map_list.hash.key << PAGE_SHIFT;
34}
35
Laurent Pincharta5ed8942013-02-17 01:54:26 +010036/*
37 * __drm_gem_cma_create - Create a GEM CMA object without allocating memory
38 * @drm: The drm device
39 * @size: The GEM object size
40 *
41 * This function creates and initializes a GEM CMA object of the given size, but
42 * doesn't allocate any memory to back the object.
43 *
44 * Return a struct drm_gem_cma_object* on success or ERR_PTR values on failure.
45 */
46static struct drm_gem_cma_object *
47__drm_gem_cma_create(struct drm_device *drm, unsigned int size)
Sascha Hauerb9d47452012-06-27 15:30:18 +020048{
Laurent Pincharta5ed8942013-02-17 01:54:26 +010049 struct drm_gem_cma_object *cma_obj;
50 struct drm_gem_object *gem_obj;
51 int ret;
52
53 cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
54 if (!cma_obj)
55 return ERR_PTR(-ENOMEM);
56
57 gem_obj = &cma_obj->base;
58
59 ret = drm_gem_object_init(drm, gem_obj, size);
60 if (ret)
61 goto error;
62
63 ret = drm_gem_create_mmap_offset(gem_obj);
64 if (ret) {
65 drm_gem_object_release(gem_obj);
66 goto error;
67 }
68
69 return cma_obj;
70
71error:
72 kfree(cma_obj);
73 return ERR_PTR(ret);
Sascha Hauerb9d47452012-06-27 15:30:18 +020074}
75
76/*
77 * drm_gem_cma_create - allocate an object with the given size
78 *
79 * returns a struct drm_gem_cma_object* on success or ERR_PTR values
80 * on failure.
81 */
82struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
83 unsigned int size)
84{
85 struct drm_gem_cma_object *cma_obj;
Laurent Pinchart71d72822013-02-17 01:57:30 +010086 struct sg_table *sgt = NULL;
87 int ret;
Sascha Hauerb9d47452012-06-27 15:30:18 +020088
89 size = round_up(size, PAGE_SIZE);
90
Laurent Pincharta5ed8942013-02-17 01:54:26 +010091 cma_obj = __drm_gem_cma_create(drm, size);
92 if (IS_ERR(cma_obj))
93 return cma_obj;
Sascha Hauerb9d47452012-06-27 15:30:18 +020094
95 cma_obj->vaddr = dma_alloc_writecombine(drm->dev, size,
96 &cma_obj->paddr, GFP_KERNEL | __GFP_NOWARN);
97 if (!cma_obj->vaddr) {
Laurent Pincharta5ed8942013-02-17 01:54:26 +010098 dev_err(drm->dev, "failed to allocate buffer with size %d\n",
99 size);
Laurent Pinchart71d72822013-02-17 01:57:30 +0100100 ret = -ENOMEM;
101 goto error;
Sascha Hauerb9d47452012-06-27 15:30:18 +0200102 }
103
Laurent Pinchart71d72822013-02-17 01:57:30 +0100104 sgt = kzalloc(sizeof(*cma_obj->sgt), GFP_KERNEL);
105 if (sgt == NULL) {
106 ret = -ENOMEM;
107 goto error;
108 }
109
110 ret = dma_get_sgtable(drm->dev, sgt, cma_obj->vaddr,
111 cma_obj->paddr, size);
112 if (ret < 0)
113 goto error;
114
115 cma_obj->sgt = sgt;
116
Sascha Hauerb9d47452012-06-27 15:30:18 +0200117 return cma_obj;
Laurent Pinchart71d72822013-02-17 01:57:30 +0100118
119error:
120 kfree(sgt);
121 drm_gem_cma_free_object(&cma_obj->base);
122 return ERR_PTR(ret);
Sascha Hauerb9d47452012-06-27 15:30:18 +0200123}
124EXPORT_SYMBOL_GPL(drm_gem_cma_create);
125
126/*
127 * drm_gem_cma_create_with_handle - allocate an object with the given
128 * size and create a gem handle on it
129 *
130 * returns a struct drm_gem_cma_object* on success or ERR_PTR values
131 * on failure.
132 */
133static struct drm_gem_cma_object *drm_gem_cma_create_with_handle(
134 struct drm_file *file_priv,
135 struct drm_device *drm, unsigned int size,
136 unsigned int *handle)
137{
138 struct drm_gem_cma_object *cma_obj;
139 struct drm_gem_object *gem_obj;
140 int ret;
141
142 cma_obj = drm_gem_cma_create(drm, size);
143 if (IS_ERR(cma_obj))
144 return cma_obj;
145
146 gem_obj = &cma_obj->base;
147
148 /*
149 * allocate a id of idr table where the obj is registered
150 * and handle has the id what user can see.
151 */
152 ret = drm_gem_handle_create(file_priv, gem_obj, handle);
153 if (ret)
154 goto err_handle_create;
155
156 /* drop reference from allocate - handle holds it now. */
157 drm_gem_object_unreference_unlocked(gem_obj);
158
159 return cma_obj;
160
161err_handle_create:
162 drm_gem_cma_free_object(gem_obj);
163
164 return ERR_PTR(ret);
165}
166
167/*
168 * drm_gem_cma_free_object - (struct drm_driver)->gem_free_object callback
169 * function
170 */
171void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
172{
173 struct drm_gem_cma_object *cma_obj;
174
175 if (gem_obj->map_list.map)
176 drm_gem_free_mmap_offset(gem_obj);
177
Sascha Hauerb9d47452012-06-27 15:30:18 +0200178 cma_obj = to_drm_gem_cma_obj(gem_obj);
179
Laurent Pinchart71d72822013-02-17 01:57:30 +0100180 if (cma_obj->vaddr) {
Laurent Pincharta5ed8942013-02-17 01:54:26 +0100181 dma_free_writecombine(gem_obj->dev->dev, cma_obj->base.size,
182 cma_obj->vaddr, cma_obj->paddr);
Laurent Pinchart71d72822013-02-17 01:57:30 +0100183 if (cma_obj->sgt) {
184 sg_free_table(cma_obj->sgt);
185 kfree(cma_obj->sgt);
186 }
187 } else if (gem_obj->import_attach) {
188 drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
189 }
Laurent Pincharta5ed8942013-02-17 01:54:26 +0100190
191 drm_gem_object_release(gem_obj);
Sascha Hauerb9d47452012-06-27 15:30:18 +0200192
193 kfree(cma_obj);
194}
195EXPORT_SYMBOL_GPL(drm_gem_cma_free_object);
196
197/*
198 * drm_gem_cma_dumb_create - (struct drm_driver)->dumb_create callback
199 * function
200 *
201 * This aligns the pitch and size arguments to the minimum required. wrap
202 * this into your own function if you need bigger alignment.
203 */
204int drm_gem_cma_dumb_create(struct drm_file *file_priv,
205 struct drm_device *dev, struct drm_mode_create_dumb *args)
206{
207 struct drm_gem_cma_object *cma_obj;
208 int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
209
210 if (args->pitch < min_pitch)
211 args->pitch = min_pitch;
212
213 if (args->size < args->pitch * args->height)
214 args->size = args->pitch * args->height;
215
216 cma_obj = drm_gem_cma_create_with_handle(file_priv, dev,
217 args->size, &args->handle);
218 if (IS_ERR(cma_obj))
219 return PTR_ERR(cma_obj);
220
221 return 0;
222}
223EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);
224
225/*
226 * drm_gem_cma_dumb_map_offset - (struct drm_driver)->dumb_map_offset callback
227 * function
228 */
229int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv,
230 struct drm_device *drm, uint32_t handle, uint64_t *offset)
231{
232 struct drm_gem_object *gem_obj;
233
234 mutex_lock(&drm->struct_mutex);
235
236 gem_obj = drm_gem_object_lookup(drm, file_priv, handle);
237 if (!gem_obj) {
238 dev_err(drm->dev, "failed to lookup gem object\n");
239 mutex_unlock(&drm->struct_mutex);
240 return -EINVAL;
241 }
242
243 *offset = get_gem_mmap_offset(gem_obj);
244
245 drm_gem_object_unreference(gem_obj);
246
247 mutex_unlock(&drm->struct_mutex);
248
249 return 0;
250}
251EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_map_offset);
252
253const struct vm_operations_struct drm_gem_cma_vm_ops = {
254 .open = drm_gem_vm_open,
255 .close = drm_gem_vm_close,
256};
257EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);
258
Laurent Pinchartebaf9e02013-04-16 14:32:34 +0200259static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj,
260 struct vm_area_struct *vma)
261{
262 int ret;
263
264 ret = remap_pfn_range(vma, vma->vm_start, cma_obj->paddr >> PAGE_SHIFT,
265 vma->vm_end - vma->vm_start, vma->vm_page_prot);
266 if (ret)
267 drm_gem_vm_close(vma);
268
269 return ret;
270}
271
Sascha Hauerb9d47452012-06-27 15:30:18 +0200272/*
273 * drm_gem_cma_mmap - (struct file_operation)->mmap callback function
274 */
275int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma)
276{
Sascha Hauerb9d47452012-06-27 15:30:18 +0200277 struct drm_gem_cma_object *cma_obj;
Laurent Pinchartebaf9e02013-04-16 14:32:34 +0200278 struct drm_gem_object *gem_obj;
Sascha Hauerb9d47452012-06-27 15:30:18 +0200279 int ret;
280
281 ret = drm_gem_mmap(filp, vma);
282 if (ret)
283 return ret;
284
285 gem_obj = vma->vm_private_data;
286 cma_obj = to_drm_gem_cma_obj(gem_obj);
287
Laurent Pinchartebaf9e02013-04-16 14:32:34 +0200288 return drm_gem_cma_mmap_obj(cma_obj, vma);
Sascha Hauerb9d47452012-06-27 15:30:18 +0200289}
290EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);
291
292/*
293 * drm_gem_cma_dumb_destroy - (struct drm_driver)->dumb_destroy callback function
294 */
295int drm_gem_cma_dumb_destroy(struct drm_file *file_priv,
296 struct drm_device *drm, unsigned int handle)
297{
298 return drm_gem_handle_delete(file_priv, handle);
299}
300EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_destroy);
Rob Clark6f646092012-12-10 10:46:43 -0600301
302#ifdef CONFIG_DEBUG_FS
303void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj, struct seq_file *m)
304{
305 struct drm_gem_object *obj = &cma_obj->base;
306 struct drm_device *dev = obj->dev;
307 uint64_t off = 0;
308
309 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
310
311 if (obj->map_list.map)
312 off = (uint64_t)obj->map_list.hash.key;
313
314 seq_printf(m, "%2d (%2d) %08llx %08Zx %p %d",
315 obj->name, obj->refcount.refcount.counter,
316 off, cma_obj->paddr, cma_obj->vaddr, obj->size);
317
318 seq_printf(m, "\n");
319}
320EXPORT_SYMBOL_GPL(drm_gem_cma_describe);
321#endif
Laurent Pinchart71d72822013-02-17 01:57:30 +0100322
323/* -----------------------------------------------------------------------------
324 * DMA-BUF
325 */
326
327struct drm_gem_cma_dmabuf_attachment {
328 struct sg_table sgt;
329 enum dma_data_direction dir;
330};
331
332static int drm_gem_cma_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev,
333 struct dma_buf_attachment *attach)
334{
335 struct drm_gem_cma_dmabuf_attachment *cma_attach;
336
337 cma_attach = kzalloc(sizeof(*cma_attach), GFP_KERNEL);
338 if (!cma_attach)
339 return -ENOMEM;
340
341 cma_attach->dir = DMA_NONE;
342 attach->priv = cma_attach;
343
344 return 0;
345}
346
347static void drm_gem_cma_dmabuf_detach(struct dma_buf *dmabuf,
348 struct dma_buf_attachment *attach)
349{
350 struct drm_gem_cma_dmabuf_attachment *cma_attach = attach->priv;
351 struct sg_table *sgt;
352
353 if (cma_attach == NULL)
354 return;
355
356 sgt = &cma_attach->sgt;
357
358 if (cma_attach->dir != DMA_NONE)
359 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
360 cma_attach->dir);
361
362 sg_free_table(sgt);
363 kfree(cma_attach);
364 attach->priv = NULL;
365}
366
367static struct sg_table *
368drm_gem_cma_dmabuf_map(struct dma_buf_attachment *attach,
369 enum dma_data_direction dir)
370{
371 struct drm_gem_cma_dmabuf_attachment *cma_attach = attach->priv;
372 struct drm_gem_cma_object *cma_obj = attach->dmabuf->priv;
373 struct drm_device *drm = cma_obj->base.dev;
374 struct scatterlist *rd, *wr;
375 struct sg_table *sgt;
376 unsigned int i;
377 int nents, ret;
378
379 DRM_DEBUG_PRIME("\n");
380
381 if (WARN_ON(dir == DMA_NONE))
382 return ERR_PTR(-EINVAL);
383
384 /* Return the cached mapping when possible. */
385 if (cma_attach->dir == dir)
386 return &cma_attach->sgt;
387
388 /* Two mappings with different directions for the same attachment are
389 * not allowed.
390 */
391 if (WARN_ON(cma_attach->dir != DMA_NONE))
392 return ERR_PTR(-EBUSY);
393
394 sgt = &cma_attach->sgt;
395
396 ret = sg_alloc_table(sgt, cma_obj->sgt->orig_nents, GFP_KERNEL);
397 if (ret) {
398 DRM_ERROR("failed to alloc sgt.\n");
399 return ERR_PTR(-ENOMEM);
400 }
401
402 mutex_lock(&drm->struct_mutex);
403
404 rd = cma_obj->sgt->sgl;
405 wr = sgt->sgl;
406 for (i = 0; i < sgt->orig_nents; ++i) {
407 sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
408 rd = sg_next(rd);
409 wr = sg_next(wr);
410 }
411
412 nents = dma_map_sg(attach->dev, sgt->sgl, sgt->orig_nents, dir);
413 if (!nents) {
414 DRM_ERROR("failed to map sgl with iommu.\n");
415 sg_free_table(sgt);
416 sgt = ERR_PTR(-EIO);
417 goto done;
418 }
419
420 cma_attach->dir = dir;
421 attach->priv = cma_attach;
422
423 DRM_DEBUG_PRIME("buffer size = %zu\n", cma_obj->base.size);
424
425done:
426 mutex_unlock(&drm->struct_mutex);
427 return sgt;
428}
429
430static void drm_gem_cma_dmabuf_unmap(struct dma_buf_attachment *attach,
431 struct sg_table *sgt,
432 enum dma_data_direction dir)
433{
434 /* Nothing to do. */
435}
436
437static void drm_gem_cma_dmabuf_release(struct dma_buf *dmabuf)
438{
439 struct drm_gem_cma_object *cma_obj = dmabuf->priv;
440
441 DRM_DEBUG_PRIME("%s\n", __FILE__);
442
443 /*
444 * drm_gem_cma_dmabuf_release() call means that file object's
445 * f_count is 0 and it calls drm_gem_object_handle_unreference()
446 * to drop the references that these values had been increased
447 * at drm_prime_handle_to_fd()
448 */
449 if (cma_obj->base.export_dma_buf == dmabuf) {
450 cma_obj->base.export_dma_buf = NULL;
451
452 /*
453 * drop this gem object refcount to release allocated buffer
454 * and resources.
455 */
456 drm_gem_object_unreference_unlocked(&cma_obj->base);
457 }
458}
459
460static void *drm_gem_cma_dmabuf_kmap_atomic(struct dma_buf *dmabuf,
461 unsigned long page_num)
462{
463 /* TODO */
464
465 return NULL;
466}
467
468static void drm_gem_cma_dmabuf_kunmap_atomic(struct dma_buf *dmabuf,
469 unsigned long page_num, void *addr)
470{
471 /* TODO */
472}
473
474static void *drm_gem_cma_dmabuf_kmap(struct dma_buf *dmabuf,
475 unsigned long page_num)
476{
477 /* TODO */
478
479 return NULL;
480}
481
482static void drm_gem_cma_dmabuf_kunmap(struct dma_buf *dmabuf,
483 unsigned long page_num, void *addr)
484{
485 /* TODO */
486}
487
488static int drm_gem_cma_dmabuf_mmap(struct dma_buf *dmabuf,
489 struct vm_area_struct *vma)
490{
491 struct drm_gem_cma_object *cma_obj = dmabuf->priv;
492 struct drm_gem_object *gem_obj = &cma_obj->base;
493 int ret;
494
495 ret = drm_gem_mmap_obj(gem_obj, gem_obj->size, vma);
496 if (ret < 0)
497 return ret;
498
499 return drm_gem_cma_mmap_obj(cma_obj, vma);
500}
501
502static void *drm_gem_cma_dmabuf_vmap(struct dma_buf *dmabuf)
503{
504 struct drm_gem_cma_object *cma_obj = dmabuf->priv;
505
506 return cma_obj->vaddr;
507}
508
509static struct dma_buf_ops drm_gem_cma_dmabuf_ops = {
510 .attach = drm_gem_cma_dmabuf_attach,
511 .detach = drm_gem_cma_dmabuf_detach,
512 .map_dma_buf = drm_gem_cma_dmabuf_map,
513 .unmap_dma_buf = drm_gem_cma_dmabuf_unmap,
514 .kmap = drm_gem_cma_dmabuf_kmap,
515 .kmap_atomic = drm_gem_cma_dmabuf_kmap_atomic,
516 .kunmap = drm_gem_cma_dmabuf_kunmap,
517 .kunmap_atomic = drm_gem_cma_dmabuf_kunmap_atomic,
518 .mmap = drm_gem_cma_dmabuf_mmap,
519 .vmap = drm_gem_cma_dmabuf_vmap,
520 .release = drm_gem_cma_dmabuf_release,
521};
522
523struct dma_buf *drm_gem_cma_dmabuf_export(struct drm_device *drm,
524 struct drm_gem_object *obj, int flags)
525{
526 struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
527
528 return dma_buf_export(cma_obj, &drm_gem_cma_dmabuf_ops,
529 cma_obj->base.size, flags);
530}
531EXPORT_SYMBOL_GPL(drm_gem_cma_dmabuf_export);
532
533struct drm_gem_object *drm_gem_cma_dmabuf_import(struct drm_device *drm,
534 struct dma_buf *dma_buf)
535{
536 struct drm_gem_cma_object *cma_obj;
537 struct dma_buf_attachment *attach;
538 struct sg_table *sgt;
539 int ret;
540
541 DRM_DEBUG_PRIME("%s\n", __FILE__);
542
543 /* is this one of own objects? */
544 if (dma_buf->ops == &drm_gem_cma_dmabuf_ops) {
545 struct drm_gem_object *obj;
546
547 cma_obj = dma_buf->priv;
548 obj = &cma_obj->base;
549
550 /* is it from our device? */
551 if (obj->dev == drm) {
552 /*
553 * Importing dmabuf exported from out own gem increases
554 * refcount on gem itself instead of f_count of dmabuf.
555 */
556 drm_gem_object_reference(obj);
557 dma_buf_put(dma_buf);
558 return obj;
559 }
560 }
561
562 /* Create a CMA GEM buffer. */
563 cma_obj = __drm_gem_cma_create(drm, dma_buf->size);
564 if (IS_ERR(cma_obj))
565 return ERR_PTR(PTR_ERR(cma_obj));
566
567 /* Attach to the buffer and map it. Make sure the mapping is contiguous
568 * on the device memory bus, as that's all we support.
569 */
570 attach = dma_buf_attach(dma_buf, drm->dev);
571 if (IS_ERR(attach)) {
572 ret = -EINVAL;
573 goto error_gem_free;
574 }
575
576 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
577 if (IS_ERR_OR_NULL(sgt)) {
578 ret = sgt ? PTR_ERR(sgt) : -ENOMEM;
579 goto error_buf_detach;
580 }
581
582 if (sgt->nents != 1) {
583 ret = -EINVAL;
584 goto error_buf_unmap;
585 }
586
587 cma_obj->base.import_attach = attach;
588 cma_obj->paddr = sg_dma_address(sgt->sgl);
589 cma_obj->sgt = sgt;
590
591 DRM_DEBUG_PRIME("dma_addr = 0x%x, size = %zu\n", cma_obj->paddr,
592 dma_buf->size);
593
594 return &cma_obj->base;
595
596error_buf_unmap:
597 dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
598error_buf_detach:
599 dma_buf_detach(dma_buf, attach);
600error_gem_free:
601 drm_gem_cma_free_object(&cma_obj->base);
602 return ERR_PTR(ret);
603}
604EXPORT_SYMBOL_GPL(drm_gem_cma_dmabuf_import);