gpu: ion: Add cache maintenance to ion.
This patch adds cache maintenance operations to ion. As per mailing
list discussions regarding dma_buf, cache operations are done implicitly.
At buffer allocaiton time the user can select whether he'd like mappings
(both kernel and user) to be cached. When cached mappings are selected,
no mappings will be created for a buffer at mmap time. Instead pages will
be faulted in one at a time so we can track which pages require flushing
before dma. When the buffers are mapped for dma (via the dma_buf apis)
any pages which were touched will be synced for device.
Change-Id: Id5d6894e8bb52af038c91dd895143bf3b4203b0b
Signed-off-by: Rebecca Schultz Zavin <rebecca@android.com>
Git-commit: e1562c62795ac013c8feaa1a707bd7c039562de9
Git-repo: https://android.googlesource.com/kernel/common
[mitchelh@codeaurora.org: mmap ops in ion: took the Google version
(plus our unmap callback). ion_carveout_heap_map_dma: took the Google
version due to different carveout mechanism. ion_heap_map_kernel: took
the CAF version due to different ioremap
mechanism. ion_system_heap_map_kernel: took the CAF version since it's
still not clear whether mapping system memory as cached is
supported. Resolved a few other merge conflicts due to the fact that
7d72bad55a0e03392d81eee77a9a8c9df6b73a9a already brought in bits and
pieces of this patch (namely the flags/heap_mask split).]
Signed-off-by: Mitchel Humpherys <mitchelh@codeaurora.org>
diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c
index a8858c0..4c2ff7d 100644
--- a/drivers/gpu/ion/ion.c
+++ b/drivers/gpu/ion/ion.c
@@ -39,7 +39,6 @@
#include <mach/iommu_domains.h>
#include "ion_priv.h"
-#define DEBUG
/**
* struct ion_device - the metadata of the ion device node
@@ -190,6 +189,8 @@
return NULL;
}
+static int ion_buffer_alloc_dirty(struct ion_buffer *buffer);
+
/* this function should only be called while dev->lock is held */
static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
struct ion_device *dev,
@@ -219,14 +220,36 @@
buffer->size = len;
buffer->flags = flags;
- table = buffer->heap->ops->map_dma(buffer->heap, buffer);
+ table = heap->ops->map_dma(heap, buffer);
if (IS_ERR_OR_NULL(table)) {
heap->ops->free(buffer);
kfree(buffer);
return ERR_PTR(PTR_ERR(table));
}
buffer->sg_table = table;
+ if (buffer->flags & ION_FLAG_CACHED)
+ for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents,
+ i) {
+ if (sg_dma_len(sg) == PAGE_SIZE)
+ continue;
+ pr_err("%s: cached mappings must have pagewise "
+ "sg_lists\n", __func__);
+ heap->ops->unmap_dma(heap, buffer);
+ kfree(buffer);
+ return ERR_PTR(-EINVAL);
+ }
+ ret = ion_buffer_alloc_dirty(buffer);
+ if (ret) {
+ heap->ops->unmap_dma(heap, buffer);
+ heap->ops->free(buffer);
+ kfree(buffer);
+ return ERR_PTR(ret);
+ }
+
+ buffer->dev = dev;
+ buffer->size = len;
+ INIT_LIST_HEAD(&buffer->vmas);
mutex_init(&buffer->lock);
/* this will set up dma addresses for the sglist -- it is not
technically correct as per the dma api -- a specific
@@ -429,6 +452,8 @@
dbg_str[0] = '\0';
+ pr_debug("%s: len %d align %d heap_mask %u flags %x\n", __func__, len,
+ align, heap_mask, flags);
/*
* traverse the list of heaps available in this system in priority
* order. If the heap type is supported by the client, and matches the
@@ -1216,12 +1241,18 @@
}
EXPORT_SYMBOL(ion_sg_table);
+static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
+ struct device *dev,
+ enum dma_data_direction direction);
+
static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct dma_buf *dmabuf = attachment->dmabuf;
struct ion_buffer *buffer = dmabuf->priv;
+ if (buffer->flags & ION_FLAG_CACHED)
+ ion_buffer_sync_for_device(buffer, attachment->dev, direction);
return buffer->sg_table;
}
@@ -1231,24 +1262,115 @@
{
}
-static void ion_vma_close(struct vm_area_struct *vma)
+static int ion_buffer_alloc_dirty(struct ion_buffer *buffer)
+{
+ unsigned long pages = buffer->sg_table->nents;
+ unsigned long length = (pages + BITS_PER_LONG - 1)/BITS_PER_LONG;
+
+ buffer->dirty = kzalloc(length * sizeof(unsigned long), GFP_KERNEL);
+ if (!buffer->dirty)
+ return -ENOMEM;
+ return 0;
+}
+
+struct ion_vma_list {
+ struct list_head list;
+ struct vm_area_struct *vma;
+};
+
+static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
+ struct device *dev,
+ enum dma_data_direction dir)
+{
+ struct scatterlist *sg;
+ int i;
+ struct ion_vma_list *vma_list;
+
+ pr_debug("%s: syncing for device %s\n", __func__,
+ dev ? dev_name(dev) : "null");
+ mutex_lock(&buffer->lock);
+ for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {
+ if (!test_bit(i, buffer->dirty))
+ continue;
+ dma_sync_sg_for_device(dev, sg, 1, dir);
+ clear_bit(i, buffer->dirty);
+ }
+ list_for_each_entry(vma_list, &buffer->vmas, list) {
+ struct vm_area_struct *vma = vma_list->vma;
+
+ zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start,
+ NULL);
+ }
+ mutex_unlock(&buffer->lock);
+}
+
+int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct ion_buffer *buffer = vma->vm_private_data;
+ struct scatterlist *sg;
+ int i;
- pr_debug("%s: %d\n", __func__, __LINE__);
+ mutex_lock(&buffer->lock);
+ set_bit(vmf->pgoff, buffer->dirty);
+
+ for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {
+ if (i != vmf->pgoff)
+ continue;
+ dma_sync_sg_for_cpu(NULL, sg, 1, DMA_BIDIRECTIONAL);
+ vm_insert_page(vma, (unsigned long)vmf->virtual_address,
+ sg_page(sg));
+ break;
+ }
+ mutex_unlock(&buffer->lock);
+ return VM_FAULT_NOPAGE;
+}
+
+static void ion_vm_open(struct vm_area_struct *vma)
+{
+ struct ion_buffer *buffer = vma->vm_private_data;
+ struct ion_vma_list *vma_list;
+
+ vma_list = kmalloc(sizeof(struct ion_vma_list), GFP_KERNEL);
+ if (!vma_list)
+ return;
+ vma_list->vma = vma;
+ mutex_lock(&buffer->lock);
+ list_add(&vma_list->list, &buffer->vmas);
+ mutex_unlock(&buffer->lock);
+ pr_debug("%s: adding %p\n", __func__, vma);
+}
+
+static void ion_vm_close(struct vm_area_struct *vma)
+{
+ struct ion_buffer *buffer = vma->vm_private_data;
+ struct ion_vma_list *vma_list, *tmp;
+
+ pr_debug("%s\n", __func__);
+ mutex_lock(&buffer->lock);
+ list_for_each_entry_safe(vma_list, tmp, &buffer->vmas, list) {
+ if (vma_list->vma != vma)
+ continue;
+ list_del(&vma_list->list);
+ kfree(vma_list);
+ pr_debug("%s: deleting %p\n", __func__, vma);
+ break;
+ }
+ mutex_unlock(&buffer->lock);
if (buffer->heap->ops->unmap_user)
buffer->heap->ops->unmap_user(buffer->heap, buffer);
}
-static struct vm_operations_struct ion_vm_ops = {
- .close = ion_vma_close,
+struct vm_operations_struct ion_vma_ops = {
+ .open = ion_vm_open,
+ .close = ion_vm_close,
+ .fault = ion_vm_fault,
};
static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
struct ion_buffer *buffer = dmabuf->priv;
- int ret;
+ int ret = 0;
if (!buffer->heap->ops->map_user) {
pr_err("%s: this heap does not define a method for mapping "
@@ -1256,22 +1378,22 @@
return -EINVAL;
}
- mutex_lock(&buffer->lock);
- /* now map it to userspace */
- ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
- mutex_unlock(&buffer->lock);
+ if (buffer->flags & ION_FLAG_CACHED) {
+ vma->vm_private_data = buffer;
+ vma->vm_ops = &ion_vma_ops;
+ ion_vm_open(vma);
+ } else {
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+ mutex_lock(&buffer->lock);
+ /* now map it to userspace */
+ ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
+ mutex_unlock(&buffer->lock);
+ }
- if (ret) {
+ if (ret)
pr_err("%s: failure mapping buffer to userspace\n",
__func__);
- } else {
- vma->vm_ops = &ion_vm_ops;
- /*
- * move the buffer into the vm_private_data so we can access it
- * from vma_open/close
- */
- vma->vm_private_data = buffer;
- }
+
return ret;
}