gpu: ion: Map a range into the IOMMU
Instead of mapping 1 4K page at a time into the IOMMU create a
scatterlist and map everything at once. This will be more efficient.
Change-Id: I8e83066869dd6f7a479bad22a66e4c70cc5973b5
Signed-off-by: Olav Haugan <ohaugan@codeaurora.org>
diff --git a/arch/arm/mach-msm/iommu_domains.c b/arch/arm/mach-msm/iommu_domains.c
index a7663b6..3c6dc64 100644
--- a/arch/arm/mach-msm/iommu_domains.c
+++ b/arch/arm/mach-msm/iommu_domains.c
@@ -13,6 +13,7 @@
#include <mach/msm_subsystem_map.h>
#include <linux/memory_alloc.h>
#include <linux/iommu.h>
+#include <linux/vmalloc.h>
#include <asm/sizes.h>
#include <asm/page.h>
#include <linux/init.h>
@@ -54,33 +55,32 @@
unsigned long size,
int cached)
{
- int i, ret;
- unsigned long temp_iova;
+ int i, ret = 0;
+ struct scatterlist *sglist;
+ unsigned int nrpages = PFN_ALIGN(size) >> PAGE_SHIFT;
+ struct page *dummy_page = phys_to_page(
+ PFN_ALIGN(virt_to_phys(iommu_dummy)));
- for (i = size, temp_iova = start_iova; i > 0; i -= SZ_4K,
- temp_iova += SZ_4K) {
- ret = iommu_map(domain, temp_iova,
- PFN_ALIGN(virt_to_phys(iommu_dummy)),
- get_order(SZ_4K),
- 0);
-
- if (ret) {
- pr_err("%s: could not map %lx to dummy page in domain"
- " %p\n",
- __func__, temp_iova, domain);
- goto out;
- }
+ sglist = vmalloc(sizeof(*sglist) * nrpages);
+ if (!sglist) {
+ ret = -ENOMEM;
+ goto err1;
}
- return 0;
+ sg_init_table(sglist, nrpages);
-out:
+ for (i = 0; i < nrpages; i++)
+ sg_set_page(&sglist[i], dummy_page, PAGE_SIZE, 0);
- for ( ; i < size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
+ ret = iommu_map_range(domain, start_iova, sglist, size, cached);
+ if (ret) {
+ pr_err("%s: could not map extra %lx in domain %p\n",
+ __func__, start_iova, domain);
+ }
- return -EINVAL;
-
+ vfree(sglist);
+err1:
+ return ret;
}
diff --git a/drivers/gpu/ion/ion_carveout_heap.c b/drivers/gpu/ion/ion_carveout_heap.c
index 7a08e50..de9e2c4 100644
--- a/drivers/gpu/ion/ion_carveout_heap.c
+++ b/drivers/gpu/ion/ion_carveout_heap.c
@@ -271,10 +271,11 @@
unsigned long iova_length,
unsigned long flags)
{
- unsigned long temp_phys, temp_iova;
struct iommu_domain *domain;
- int i, ret = 0;
+ int ret = 0;
unsigned long extra;
+ int prot = ION_IS_CACHED(flags) ? 1 : 0;
+ struct scatterlist *sglist = 0;
data->mapped_size = iova_length;
@@ -300,32 +301,36 @@
goto out1;
}
- temp_iova = data->iova_addr;
- temp_phys = buffer->priv_phys;
- for (i = buffer->size; i > 0; i -= SZ_4K, temp_iova += SZ_4K,
- temp_phys += SZ_4K) {
- ret = iommu_map(domain, temp_iova, temp_phys,
- get_order(SZ_4K),
- ION_IS_CACHED(flags) ? 1 : 0);
+ sglist = vmalloc(sizeof(*sglist));
+ if (!sglist)
+ goto out1;
- if (ret) {
- pr_err("%s: could not map %lx to %lx in domain %p\n",
- __func__, temp_iova, temp_phys, domain);
- goto out2;
- }
+ sg_init_table(sglist, 1);
+ sglist->length = buffer->size;
+ sglist->offset = 0;
+ sglist->dma_address = buffer->priv_phys;
+
+ ret = iommu_map_range(domain, data->iova_addr, sglist,
+ buffer->size, prot);
+ if (ret) {
+ pr_err("%s: could not map %lx in domain %p\n",
+ __func__, data->iova_addr, domain);
+ goto out1;
}
- if (extra && (msm_iommu_map_extra(domain, temp_iova, extra, flags) < 0))
- goto out2;
-
- return 0;
-
+ if (extra) {
+ unsigned long extra_iova_addr = data->iova_addr + buffer->size;
+ ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, prot);
+ if (ret)
+ goto out2;
+ }
+ vfree(sglist);
+ return ret;
out2:
- for ( ; i < buffer->size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
+ vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
@@ -336,8 +341,6 @@
void ion_carveout_heap_unmap_iommu(struct ion_iommu_map *data)
{
- int i;
- unsigned long temp_iova;
unsigned int domain_num;
unsigned int partition_num;
struct iommu_domain *domain;
@@ -355,10 +358,7 @@
return;
}
- temp_iova = data->iova_addr;
- for (i = data->mapped_size; i > 0; i -= SZ_4K, temp_iova += SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, data->mapped_size);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
diff --git a/drivers/gpu/ion/ion_cp_heap.c b/drivers/gpu/ion/ion_cp_heap.c
index 85c0534..7d99482 100644
--- a/drivers/gpu/ion/ion_cp_heap.c
+++ b/drivers/gpu/ion/ion_cp_heap.c
@@ -295,8 +295,7 @@
buffer->priv_phys = ION_CP_ALLOCATE_FAIL;
}
-struct scatterlist *ion_cp_heap_map_dma(struct ion_heap *heap,
- struct ion_buffer *buffer)
+struct scatterlist *ion_cp_heap_create_sglist(struct ion_buffer *buffer)
{
struct scatterlist *sglist;
@@ -312,6 +311,12 @@
return sglist;
}
+struct scatterlist *ion_cp_heap_map_dma(struct ion_heap *heap,
+ struct ion_buffer *buffer)
+{
+ return ion_cp_heap_create_sglist(buffer);
+}
+
void ion_cp_heap_unmap_dma(struct ion_heap *heap,
struct ion_buffer *buffer)
{
@@ -570,10 +575,11 @@
unsigned long iova_length,
unsigned long flags)
{
- unsigned long temp_phys, temp_iova;
struct iommu_domain *domain;
- int i, ret = 0;
+ int ret = 0;
unsigned long extra;
+ int prot = ION_IS_CACHED(flags) ? 1 : 0;
+ struct scatterlist *sglist = 0;
data->mapped_size = iova_length;
@@ -599,30 +605,33 @@
goto out1;
}
- temp_iova = data->iova_addr;
- temp_phys = buffer->priv_phys;
- for (i = buffer->size; i > 0; i -= SZ_4K, temp_iova += SZ_4K,
- temp_phys += SZ_4K) {
- ret = iommu_map(domain, temp_iova, temp_phys,
- get_order(SZ_4K),
- ION_IS_CACHED(flags) ? 1 : 0);
-
- if (ret) {
- pr_err("%s: could not map %lx to %lx in domain %p\n",
- __func__, temp_iova, temp_phys, domain);
- goto out2;
- }
+ sglist = ion_cp_heap_create_sglist(buffer);
+ if (IS_ERR_OR_NULL(sglist)) {
+ ret = -ENOMEM;
+ goto out1;
+ }
+ ret = iommu_map_range(domain, data->iova_addr, sglist,
+ buffer->size, prot);
+ if (ret) {
+ pr_err("%s: could not map %lx in domain %p\n",
+ __func__, data->iova_addr, domain);
+ goto out1;
}
- if (extra && (msm_iommu_map_extra(domain, temp_iova, extra, flags) < 0))
- goto out2;
-
- return 0;
+ if (extra) {
+ unsigned long extra_iova_addr = data->iova_addr + buffer->size;
+ ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, prot);
+ if (ret)
+ goto out2;
+ }
+ vfree(sglist);
+ return ret;
out2:
- for ( ; i < buffer->size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
+ iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
+ if (!IS_ERR_OR_NULL(sglist))
+ vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out:
@@ -631,8 +640,6 @@
static void ion_cp_heap_unmap_iommu(struct ion_iommu_map *data)
{
- int i;
- unsigned long temp_iova;
unsigned int domain_num;
unsigned int partition_num;
struct iommu_domain *domain;
@@ -650,10 +657,7 @@
return;
}
- temp_iova = data->iova_addr;
- for (i = data->mapped_size; i > 0; i -= SZ_4K, temp_iova += SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, data->mapped_size);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
diff --git a/drivers/gpu/ion/ion_iommu_heap.c b/drivers/gpu/ion/ion_iommu_heap.c
index e754496..baf0a66 100644
--- a/drivers/gpu/ion/ion_iommu_heap.c
+++ b/drivers/gpu/ion/ion_iommu_heap.c
@@ -33,6 +33,7 @@
struct page **pages;
int nrpages;
unsigned long size;
+ struct scatterlist *iommu_sglist;
};
static int ion_iommu_heap_allocate(struct ion_heap *heap,
@@ -56,11 +57,22 @@
ret = -ENOMEM;
goto err1;
}
+ data->iommu_sglist = vmalloc(sizeof(*data->iommu_sglist) *
+ data->nrpages);
+ if (!data->iommu_sglist) {
+ ret = -ENOMEM;
+ goto err1;
+ }
+
+ sg_init_table(data->iommu_sglist, data->nrpages);
for (i = 0; i < data->nrpages; i++) {
data->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!data->pages[i])
goto err2;
+
+ sg_set_page(&data->iommu_sglist[i], data->pages[i],
+ PAGE_SIZE, 0);
}
@@ -73,6 +85,9 @@
err2:
+ vfree(data->iommu_sglist);
+ data->iommu_sglist = NULL;
+
for (i = 0; i < data->nrpages; i++) {
if (data->pages[i])
__free_page(data->pages[i]);
@@ -94,6 +109,9 @@
for (i = 0; i < data->nrpages; i++)
__free_page(data->pages[i]);
+ vfree(data->iommu_sglist);
+ data->iommu_sglist = NULL;
+
kfree(data->pages);
kfree(data);
}
@@ -160,11 +178,11 @@
unsigned long iova_length,
unsigned long flags)
{
- unsigned long temp_iova;
struct iommu_domain *domain;
- struct ion_iommu_priv_data *buffer_data = buffer->priv_virt;
- int i, j, ret = 0;
+ int ret = 0;
unsigned long extra;
+ int prot = ION_IS_CACHED(flags) ? 1 : 0;
+ struct ion_iommu_priv_data *buffer_data = buffer->priv_virt;
BUG_ON(!msm_use_iommu());
@@ -186,36 +204,24 @@
goto out1;
}
- temp_iova = data->iova_addr;
- for (i = buffer->size, j = 0; i > 0; j++, i -= SZ_4K,
- temp_iova += SZ_4K) {
- ret = iommu_map(domain, temp_iova,
- page_to_phys(buffer_data->pages[j]),
- get_order(SZ_4K),
- ION_IS_CACHED(flags) ? 1 : 0);
-
- if (ret) {
- pr_err("%s: could not map %lx to %x in domain %p\n",
- __func__, temp_iova,
- page_to_phys(buffer_data->pages[j]),
- domain);
- goto out2;
- }
+ ret = iommu_map_range(domain, data->iova_addr,
+ buffer_data->iommu_sglist, buffer->size, prot);
+ if (ret) {
+ pr_err("%s: could not map %lx in domain %p\n",
+ __func__, data->iova_addr, domain);
+ goto out1;
}
-
- if (extra &&
- msm_iommu_map_extra
- (domain, temp_iova, extra, flags) < 0)
- goto out2;
-
- return 0;
-
+ if (extra) {
+ unsigned long extra_iova_addr = data->iova_addr + buffer->size;
+ ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, prot);
+ if (ret)
+ goto out2;
+ }
+ return ret;
out2:
- for ( ; i < buffer->size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
buffer->size);
@@ -227,8 +233,6 @@
void ion_iommu_heap_unmap_iommu(struct ion_iommu_map *data)
{
- int i;
- unsigned long temp_iova;
unsigned int domain_num;
unsigned int partition_num;
struct iommu_domain *domain;
@@ -245,10 +249,7 @@
return;
}
- temp_iova = data->iova_addr;
- for (i = data->mapped_size; i > 0; i -= SZ_4K, temp_iova += SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, data->mapped_size);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
@@ -293,30 +294,13 @@
static struct scatterlist *ion_iommu_heap_map_dma(struct ion_heap *heap,
struct ion_buffer *buffer)
{
- struct scatterlist *sglist = NULL;
- if (buffer->priv_virt) {
- struct ion_iommu_priv_data *data = buffer->priv_virt;
- unsigned int i;
-
- if (!data->nrpages)
- return NULL;
-
- sglist = vmalloc(sizeof(*sglist) * data->nrpages);
- if (!sglist)
- return ERR_PTR(-ENOMEM);
-
- sg_init_table(sglist, data->nrpages);
- for (i = 0; i < data->nrpages; ++i)
- sg_set_page(&sglist[i], data->pages[i], PAGE_SIZE, 0);
- }
- return sglist;
+ struct ion_iommu_priv_data *data = buffer->priv_virt;
+ return data->iommu_sglist;
}
static void ion_iommu_heap_unmap_dma(struct ion_heap *heap,
struct ion_buffer *buffer)
{
- if (buffer->sglist)
- vfree(buffer->sglist);
}
static struct ion_heap_ops iommu_heap_ops = {
diff --git a/drivers/gpu/ion/ion_system_heap.c b/drivers/gpu/ion/ion_system_heap.c
index 316740e..bc288e7 100644
--- a/drivers/gpu/ion/ion_system_heap.c
+++ b/drivers/gpu/ion/ion_system_heap.c
@@ -104,8 +104,6 @@
void ion_system_heap_unmap_iommu(struct ion_iommu_map *data)
{
- int i;
- unsigned long temp_iova;
unsigned int domain_num;
unsigned int partition_num;
struct iommu_domain *domain;
@@ -123,10 +121,7 @@
return;
}
- temp_iova = data->iova_addr;
- for (i = data->mapped_size; i > 0; i -= SZ_4K, temp_iova += SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, data->mapped_size);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
@@ -206,11 +201,15 @@
unsigned long iova_length,
unsigned long flags)
{
- int ret, i;
- unsigned long temp_iova;
+ int ret = 0, i;
struct iommu_domain *domain;
- void *temp_phys;
unsigned long extra;
+ unsigned long extra_iova_addr;
+ struct page *page;
+ int npages = buffer->size >> PAGE_SHIFT;
+ void *vaddr = buffer->priv_virt;
+ struct scatterlist *sglist = 0;
+ int prot = ION_IS_CACHED(flags) ? 1 : 0;
if (!ION_IS_CACHED(flags))
return -EINVAL;
@@ -236,35 +235,46 @@
goto out1;
}
- temp_iova = data->iova_addr;
- temp_phys = buffer->vaddr;
- for (i = buffer->size; i > 0; i -= SZ_4K, temp_iova += SZ_4K,
- temp_phys += SZ_4K) {
- ret = iommu_map(domain, temp_iova,
- page_to_phys(vmalloc_to_page(temp_phys)),
- get_order(SZ_4K), ION_IS_CACHED(flags) ? 1 : 0);
- if (ret) {
- pr_err("%s: could not map %lx to %x in domain %p\n",
- __func__, temp_iova,
- page_to_phys(vmalloc_to_page(temp_phys)),
- domain);
- goto out2;
- }
+ sglist = vmalloc(sizeof(*sglist) * npages);
+ if (!sglist) {
+ ret = -ENOMEM;
+ goto out1;
}
- if (extra && (msm_iommu_map_extra(domain, temp_iova, extra, flags) < 0))
- goto out2;
+ sg_init_table(sglist, npages);
+ for (i = 0; i < npages; i++) {
+ page = vmalloc_to_page(vaddr);
+ if (!page)
+ goto out1;
+ sg_set_page(&sglist[i], page, PAGE_SIZE, 0);
+ vaddr += PAGE_SIZE;
+ }
- return 0;
+ ret = iommu_map_range(domain, data->iova_addr, sglist,
+ buffer->size, prot);
+
+ if (ret) {
+ pr_err("%s: could not map %lx in domain %p\n",
+ __func__, data->iova_addr, domain);
+ goto out1;
+ }
+
+ extra_iova_addr = data->iova_addr + buffer->size;
+ if (extra) {
+ ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, prot);
+ if (ret)
+ goto out2;
+ }
+ vfree(sglist);
+ return ret;
out2:
- for ( ; i < buffer->size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
-
+ iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
+ vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
- data->mapped_size);
+ data->mapped_size);
out:
return ret;
}
@@ -408,10 +418,12 @@
unsigned long iova_length,
unsigned long flags)
{
- int ret, i;
+ int ret = 0;
struct iommu_domain *domain;
- unsigned long temp_phys, temp_iova;
unsigned long extra;
+ int prot = ION_IS_CACHED(flags) ? 1 : 0;
+ struct scatterlist *sglist = 0;
+ struct page *page = 0;
if (!ION_IS_CACHED(flags))
return -EINVAL;
@@ -438,30 +450,36 @@
ret = -ENOMEM;
goto out1;
}
- temp_iova = data->iova_addr;
- temp_phys = virt_to_phys(buffer->vaddr);
- for (i = buffer->size; i > 0; i -= SZ_4K, temp_iova += SZ_4K,
- temp_phys += SZ_4K) {
- ret = iommu_map(domain, temp_iova,
- temp_phys,
- get_order(SZ_4K), ION_IS_CACHED(flags) ? 1 : 0);
+ page = virt_to_page(buffer->vaddr);
- if (ret) {
- pr_err("%s: could not map %lx to %lx in domain %p\n",
- __func__, temp_iova, temp_phys, domain);
- goto out2;
- }
+ sglist = vmalloc(sizeof(*sglist));
+ if (!sglist)
+ goto out1;
+
+ sg_init_table(sglist, 1);
+ sg_set_page(sglist, page, buffer->size, 0);
+
+ ret = iommu_map_range(domain, data->iova_addr, sglist,
+ buffer->size, prot);
+ if (ret) {
+ pr_err("%s: could not map %lx in domain %p\n",
+ __func__, data->iova_addr, domain);
+ goto out1;
}
- if (extra && (msm_iommu_map_extra(domain, temp_iova, extra, flags) < 0))
- goto out2;
-
- return 0;
+ if (extra) {
+ unsigned long extra_iova_addr = data->iova_addr + buffer->size;
+ ret = msm_iommu_map_extra(domain, extra_iova_addr, extra, prot);
+ if (ret)
+ goto out2;
+ }
+ vfree(sglist);
+ return ret;
out2:
- for ( ; i < buffer->size; i += SZ_4K, temp_iova -= SZ_4K)
- iommu_unmap(domain, temp_iova, get_order(SZ_4K));
+ iommu_unmap_range(domain, data->iova_addr, buffer->size);
out1:
+ vfree(sglist);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
out: