Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013 Red Hat |
| 3 | * Author: Rob Clark <robdclark@gmail.com> |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 as published by |
| 7 | * the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program. If not, see <http://www.gnu.org/licenses/>. |
| 16 | */ |
| 17 | |
| 18 | #include <linux/spinlock.h> |
| 19 | #include <linux/shmem_fs.h> |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 20 | #include <linux/dma-buf.h> |
Dan Williams | 01c8f1c | 2016-01-15 16:56:40 -0800 | [diff] [blame] | 21 | #include <linux/pfn_t.h> |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 22 | |
| 23 | #include "msm_drv.h" |
Rob Clark | fde5de6 | 2016-03-15 15:35:08 -0400 | [diff] [blame] | 24 | #include "msm_fence.h" |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 25 | #include "msm_gem.h" |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 26 | #include "msm_gpu.h" |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 27 | #include "msm_mmu.h" |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 28 | |
Jordan Crouse | 12bf362 | 2017-02-13 10:14:11 -0700 | [diff] [blame] | 29 | static void *get_dmabuf_ptr(struct drm_gem_object *obj) |
| 30 | { |
| 31 | return (obj && obj->import_attach) ? obj->import_attach->dmabuf : NULL; |
| 32 | } |
| 33 | |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 34 | static dma_addr_t physaddr(struct drm_gem_object *obj) |
| 35 | { |
| 36 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 37 | struct msm_drm_private *priv = obj->dev->dev_private; |
| 38 | return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) + |
| 39 | priv->vram.paddr; |
| 40 | } |
| 41 | |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 42 | static bool use_pages(struct drm_gem_object *obj) |
| 43 | { |
| 44 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 45 | return !msm_obj->vram_node; |
| 46 | } |
| 47 | |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 48 | /* allocate pages from VRAM carveout, used when no IOMMU: */ |
| 49 | static struct page **get_pages_vram(struct drm_gem_object *obj, |
| 50 | int npages) |
| 51 | { |
| 52 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 53 | struct msm_drm_private *priv = obj->dev->dev_private; |
| 54 | dma_addr_t paddr; |
| 55 | struct page **p; |
| 56 | int ret, i; |
| 57 | |
| 58 | p = drm_malloc_ab(npages, sizeof(struct page *)); |
| 59 | if (!p) |
| 60 | return ERR_PTR(-ENOMEM); |
| 61 | |
| 62 | ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, |
| 63 | npages, 0, DRM_MM_SEARCH_DEFAULT); |
| 64 | if (ret) { |
| 65 | drm_free_large(p); |
| 66 | return ERR_PTR(ret); |
| 67 | } |
| 68 | |
| 69 | paddr = physaddr(obj); |
| 70 | for (i = 0; i < npages; i++) { |
| 71 | p[i] = phys_to_page(paddr); |
| 72 | paddr += PAGE_SIZE; |
| 73 | } |
| 74 | |
| 75 | return p; |
| 76 | } |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 77 | |
| 78 | /* called with dev->struct_mutex held */ |
| 79 | static struct page **get_pages(struct drm_gem_object *obj) |
| 80 | { |
| 81 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 82 | |
| 83 | if (!msm_obj->pages) { |
| 84 | struct drm_device *dev = obj->dev; |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 85 | struct page **p; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 86 | int npages = obj->size >> PAGE_SHIFT; |
| 87 | |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 88 | if (use_pages(obj)) |
David Herrmann | 0cdbe8a | 2014-05-25 12:59:47 +0200 | [diff] [blame] | 89 | p = drm_gem_get_pages(obj); |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 90 | else |
| 91 | p = get_pages_vram(obj, npages); |
| 92 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 93 | if (IS_ERR(p)) { |
| 94 | dev_err(dev->dev, "could not get pages: %ld\n", |
| 95 | PTR_ERR(p)); |
| 96 | return p; |
| 97 | } |
| 98 | |
| 99 | msm_obj->sgt = drm_prime_pages_to_sg(p, npages); |
Wei Yongjun | 1f70e07 | 2013-09-11 06:56:12 +0800 | [diff] [blame] | 100 | if (IS_ERR(msm_obj->sgt)) { |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 101 | dev_err(dev->dev, "failed to allocate sgt\n"); |
Wei Yongjun | 1f70e07 | 2013-09-11 06:56:12 +0800 | [diff] [blame] | 102 | return ERR_CAST(msm_obj->sgt); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 103 | } |
| 104 | |
| 105 | msm_obj->pages = p; |
| 106 | |
Jordan Crouse | d162c8c | 2017-02-07 09:03:40 -0700 | [diff] [blame] | 107 | /* |
| 108 | * Make sure to flush the CPU cache for newly allocated memory |
| 109 | * so we don't get ourselves into trouble with a dirty cache |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 110 | */ |
| 111 | if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED)) |
Jordan Crouse | d162c8c | 2017-02-07 09:03:40 -0700 | [diff] [blame] | 112 | dma_sync_sg_for_device(dev->dev, msm_obj->sgt->sgl, |
| 113 | msm_obj->sgt->nents, DMA_BIDIRECTIONAL); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 114 | } |
| 115 | |
| 116 | return msm_obj->pages; |
| 117 | } |
| 118 | |
| 119 | static void put_pages(struct drm_gem_object *obj) |
| 120 | { |
| 121 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 122 | |
| 123 | if (msm_obj->pages) { |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 124 | sg_free_table(msm_obj->sgt); |
| 125 | kfree(msm_obj->sgt); |
| 126 | |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 127 | if (use_pages(obj)) |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 128 | drm_gem_put_pages(obj, msm_obj->pages, true, false); |
Micah Richert | 1ffa242 | 2014-04-09 14:11:31 -0700 | [diff] [blame] | 129 | else { |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 130 | drm_mm_remove_node(msm_obj->vram_node); |
Micah Richert | 1ffa242 | 2014-04-09 14:11:31 -0700 | [diff] [blame] | 131 | drm_free_large(msm_obj->pages); |
| 132 | } |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 133 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 134 | msm_obj->pages = NULL; |
| 135 | } |
| 136 | } |
| 137 | |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 138 | struct page **msm_gem_get_pages(struct drm_gem_object *obj) |
| 139 | { |
| 140 | struct drm_device *dev = obj->dev; |
| 141 | struct page **p; |
| 142 | mutex_lock(&dev->struct_mutex); |
| 143 | p = get_pages(obj); |
| 144 | mutex_unlock(&dev->struct_mutex); |
| 145 | return p; |
| 146 | } |
| 147 | |
| 148 | void msm_gem_put_pages(struct drm_gem_object *obj) |
| 149 | { |
| 150 | /* when we start tracking the pin count, then do something here */ |
| 151 | } |
| 152 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 153 | int msm_gem_mmap_obj(struct drm_gem_object *obj, |
| 154 | struct vm_area_struct *vma) |
| 155 | { |
| 156 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 157 | |
| 158 | vma->vm_flags &= ~VM_PFNMAP; |
| 159 | vma->vm_flags |= VM_MIXEDMAP; |
| 160 | |
| 161 | if (msm_obj->flags & MSM_BO_WC) { |
| 162 | vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); |
| 163 | } else if (msm_obj->flags & MSM_BO_UNCACHED) { |
| 164 | vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); |
| 165 | } else { |
| 166 | /* |
| 167 | * Shunt off cached objs to shmem file so they have their own |
| 168 | * address_space (so unmap_mapping_range does what we want, |
| 169 | * in particular in the case of mmap'd dmabufs) |
| 170 | */ |
| 171 | fput(vma->vm_file); |
| 172 | get_file(obj->filp); |
| 173 | vma->vm_pgoff = 0; |
| 174 | vma->vm_file = obj->filp; |
| 175 | |
| 176 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); |
| 177 | } |
| 178 | |
| 179 | return 0; |
| 180 | } |
| 181 | |
| 182 | int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma) |
| 183 | { |
| 184 | int ret; |
| 185 | |
| 186 | ret = drm_gem_mmap(filp, vma); |
| 187 | if (ret) { |
| 188 | DBG("mmap failed: %d", ret); |
| 189 | return ret; |
| 190 | } |
| 191 | |
| 192 | return msm_gem_mmap_obj(vma->vm_private_data, vma); |
| 193 | } |
| 194 | |
| 195 | int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
| 196 | { |
| 197 | struct drm_gem_object *obj = vma->vm_private_data; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 198 | struct drm_device *dev = obj->dev; |
Rob Clark | d78d383 | 2016-08-22 15:28:38 -0400 | [diff] [blame] | 199 | struct msm_drm_private *priv = dev->dev_private; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 200 | struct page **pages; |
| 201 | unsigned long pfn; |
| 202 | pgoff_t pgoff; |
| 203 | int ret; |
| 204 | |
Rob Clark | d78d383 | 2016-08-22 15:28:38 -0400 | [diff] [blame] | 205 | /* This should only happen if userspace tries to pass a mmap'd |
| 206 | * but unfaulted gem bo vaddr into submit ioctl, triggering |
| 207 | * a page fault while struct_mutex is already held. This is |
| 208 | * not a valid use-case so just bail. |
| 209 | */ |
| 210 | if (priv->struct_mutex_task == current) |
| 211 | return VM_FAULT_SIGBUS; |
| 212 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 213 | /* Make sure we don't parallel update on a fault, nor move or remove |
| 214 | * something from beneath our feet |
| 215 | */ |
| 216 | ret = mutex_lock_interruptible(&dev->struct_mutex); |
| 217 | if (ret) |
| 218 | goto out; |
| 219 | |
| 220 | /* make sure we have pages attached now */ |
| 221 | pages = get_pages(obj); |
| 222 | if (IS_ERR(pages)) { |
| 223 | ret = PTR_ERR(pages); |
| 224 | goto out_unlock; |
| 225 | } |
| 226 | |
| 227 | /* We don't use vmf->pgoff since that has the fake offset: */ |
| 228 | pgoff = ((unsigned long)vmf->virtual_address - |
| 229 | vma->vm_start) >> PAGE_SHIFT; |
| 230 | |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 231 | pfn = page_to_pfn(pages[pgoff]); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 232 | |
| 233 | VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address, |
| 234 | pfn, pfn << PAGE_SHIFT); |
| 235 | |
Dan Williams | 01c8f1c | 2016-01-15 16:56:40 -0800 | [diff] [blame] | 236 | ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, |
| 237 | __pfn_to_pfn_t(pfn, PFN_DEV)); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 238 | |
| 239 | out_unlock: |
| 240 | mutex_unlock(&dev->struct_mutex); |
| 241 | out: |
| 242 | switch (ret) { |
| 243 | case -EAGAIN: |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 244 | case 0: |
| 245 | case -ERESTARTSYS: |
| 246 | case -EINTR: |
Rob Clark | 505886d | 2013-10-20 11:57:52 -0400 | [diff] [blame] | 247 | case -EBUSY: |
| 248 | /* |
| 249 | * EBUSY is ok: this just means that another thread |
| 250 | * already did the job. |
| 251 | */ |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 252 | return VM_FAULT_NOPAGE; |
| 253 | case -ENOMEM: |
| 254 | return VM_FAULT_OOM; |
| 255 | default: |
| 256 | return VM_FAULT_SIGBUS; |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | /** get mmap offset */ |
| 261 | static uint64_t mmap_offset(struct drm_gem_object *obj) |
| 262 | { |
| 263 | struct drm_device *dev = obj->dev; |
| 264 | int ret; |
| 265 | |
| 266 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 267 | |
| 268 | /* Make it mmapable */ |
| 269 | ret = drm_gem_create_mmap_offset(obj); |
| 270 | |
| 271 | if (ret) { |
| 272 | dev_err(dev->dev, "could not allocate mmap offset\n"); |
| 273 | return 0; |
| 274 | } |
| 275 | |
| 276 | return drm_vma_node_offset_addr(&obj->vma_node); |
| 277 | } |
| 278 | |
| 279 | uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj) |
| 280 | { |
| 281 | uint64_t offset; |
| 282 | mutex_lock(&obj->dev->struct_mutex); |
| 283 | offset = mmap_offset(obj); |
| 284 | mutex_unlock(&obj->dev->struct_mutex); |
| 285 | return offset; |
| 286 | } |
| 287 | |
Abhijit Kulkarni | 1b3340c | 2017-06-22 12:39:37 -0700 | [diff] [blame] | 288 | dma_addr_t msm_gem_get_dma_addr(struct drm_gem_object *obj) |
| 289 | { |
| 290 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 291 | struct drm_device *dev = obj->dev; |
| 292 | |
| 293 | if (IS_ERR_OR_NULL(msm_obj->sgt)) { |
| 294 | dev_err(dev->dev, "invalid scatter/gather table\n"); |
| 295 | return 0; |
| 296 | } |
| 297 | |
| 298 | return sg_dma_address(msm_obj->sgt->sgl); |
| 299 | } |
| 300 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 301 | static void obj_remove_domain(struct msm_gem_vma *domain) |
| 302 | { |
| 303 | if (domain) { |
| 304 | list_del(&domain->list); |
| 305 | kfree(domain); |
| 306 | } |
| 307 | } |
| 308 | |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 309 | static void put_iova(struct drm_gem_object *obj) |
Rob Clark | 4fe5f65 | 2016-06-01 11:38:28 -0400 | [diff] [blame] | 310 | { |
| 311 | struct drm_device *dev = obj->dev; |
Rob Clark | 4fe5f65 | 2016-06-01 11:38:28 -0400 | [diff] [blame] | 312 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 313 | struct msm_gem_vma *domain, *tmp; |
Rob Clark | 4fe5f65 | 2016-06-01 11:38:28 -0400 | [diff] [blame] | 314 | |
| 315 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 316 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 317 | list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) { |
| 318 | if (iommu_present(&platform_bus_type)) { |
| 319 | msm_gem_unmap_vma(domain->aspace, domain, |
| 320 | msm_obj->sgt, get_dmabuf_ptr(obj)); |
| 321 | } |
| 322 | |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 323 | /* |
| 324 | * put_iova removes the domain connected to the obj which makes |
| 325 | * the aspace inaccessible. Store the aspace, as it is used to |
| 326 | * update the active_list during gem_free_obj and gem_purege. |
| 327 | */ |
| 328 | msm_obj->aspace = domain->aspace; |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 329 | obj_remove_domain(domain); |
Rob Clark | 4fe5f65 | 2016-06-01 11:38:28 -0400 | [diff] [blame] | 330 | } |
| 331 | } |
| 332 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 333 | static struct msm_gem_vma *obj_add_domain(struct drm_gem_object *obj, |
| 334 | struct msm_gem_address_space *aspace) |
| 335 | { |
| 336 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 337 | struct msm_gem_vma *domain = kzalloc(sizeof(*domain), GFP_KERNEL); |
| 338 | |
| 339 | if (!domain) |
| 340 | return ERR_PTR(-ENOMEM); |
| 341 | |
| 342 | domain->aspace = aspace; |
| 343 | |
| 344 | list_add_tail(&domain->list, &msm_obj->domains); |
| 345 | |
| 346 | return domain; |
| 347 | } |
| 348 | |
| 349 | static struct msm_gem_vma *obj_get_domain(struct drm_gem_object *obj, |
| 350 | struct msm_gem_address_space *aspace) |
| 351 | { |
| 352 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 353 | struct msm_gem_vma *domain; |
| 354 | |
| 355 | list_for_each_entry(domain, &msm_obj->domains, list) { |
| 356 | if (domain->aspace == aspace) |
| 357 | return domain; |
| 358 | } |
| 359 | |
| 360 | return NULL; |
| 361 | } |
| 362 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 363 | /* should be called under struct_mutex.. although it can be called |
| 364 | * from atomic context without struct_mutex to acquire an extra |
| 365 | * iova ref if you know one is already held. |
| 366 | * |
| 367 | * That means when I do eventually need to add support for unpinning |
| 368 | * the refcnt counter needs to be atomic_t. |
| 369 | */ |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 370 | int msm_gem_get_iova_locked(struct drm_gem_object *obj, |
| 371 | struct msm_gem_address_space *aspace, uint32_t *iova) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 372 | { |
| 373 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 374 | struct page **pages; |
| 375 | struct msm_gem_vma *domain; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 376 | int ret = 0; |
| 377 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 378 | if (!iommu_present(&platform_bus_type)) { |
| 379 | pages = get_pages(obj); |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 380 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 381 | if (IS_ERR(pages)) |
| 382 | return PTR_ERR(pages); |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 383 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 384 | *iova = physaddr(obj); |
| 385 | return 0; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 386 | } |
| 387 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 388 | domain = obj_get_domain(obj, aspace); |
| 389 | |
| 390 | if (!domain) { |
| 391 | domain = obj_add_domain(obj, aspace); |
| 392 | if (IS_ERR(domain)) |
| 393 | return PTR_ERR(domain); |
| 394 | |
| 395 | pages = get_pages(obj); |
| 396 | if (IS_ERR(pages)) { |
| 397 | obj_remove_domain(domain); |
| 398 | return PTR_ERR(pages); |
| 399 | } |
| 400 | |
| 401 | ret = msm_gem_map_vma(aspace, domain, msm_obj->sgt, |
| 402 | get_dmabuf_ptr(obj), |
| 403 | msm_obj->flags); |
| 404 | } |
| 405 | |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 406 | if (!ret && domain) { |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 407 | *iova = domain->iova; |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 408 | if (aspace && aspace->domain_attached) |
| 409 | msm_gem_add_obj_to_aspace_active_list(aspace, obj); |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 410 | } else { |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 411 | obj_remove_domain(domain); |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 412 | } |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 413 | |
| 414 | return ret; |
| 415 | } |
| 416 | |
Rob Clark | 2638d90 | 2014-11-08 09:13:37 -0500 | [diff] [blame] | 417 | /* get iova, taking a reference. Should have a matching put */ |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 418 | int msm_gem_get_iova(struct drm_gem_object *obj, |
| 419 | struct msm_gem_address_space *aspace, uint32_t *iova) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 420 | { |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 421 | struct msm_gem_vma *domain; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 422 | int ret; |
Rob Clark | edd4fc6 | 2013-09-14 14:01:55 -0400 | [diff] [blame] | 423 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 424 | domain = obj_get_domain(obj, aspace); |
| 425 | if (domain) { |
| 426 | *iova = domain->iova; |
Rob Clark | edd4fc6 | 2013-09-14 14:01:55 -0400 | [diff] [blame] | 427 | return 0; |
| 428 | } |
| 429 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 430 | mutex_lock(&obj->dev->struct_mutex); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 431 | ret = msm_gem_get_iova_locked(obj, aspace, iova); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 432 | mutex_unlock(&obj->dev->struct_mutex); |
| 433 | return ret; |
| 434 | } |
| 435 | |
Rob Clark | 2638d90 | 2014-11-08 09:13:37 -0500 | [diff] [blame] | 436 | /* get iova without taking a reference, used in places where you have |
| 437 | * already done a 'msm_gem_get_iova()'. |
| 438 | */ |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 439 | uint32_t msm_gem_iova(struct drm_gem_object *obj, |
| 440 | struct msm_gem_address_space *aspace) |
Rob Clark | 2638d90 | 2014-11-08 09:13:37 -0500 | [diff] [blame] | 441 | { |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 442 | struct msm_gem_vma *domain = obj_get_domain(obj, aspace); |
| 443 | |
| 444 | WARN_ON(!domain); |
| 445 | |
| 446 | return domain ? domain->iova : 0; |
Rob Clark | 2638d90 | 2014-11-08 09:13:37 -0500 | [diff] [blame] | 447 | } |
| 448 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 449 | void msm_gem_put_iova(struct drm_gem_object *obj, |
| 450 | struct msm_gem_address_space *aspace) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 451 | { |
| 452 | // XXX TODO .. |
| 453 | // NOTE: probably don't need a _locked() version.. we wouldn't |
| 454 | // normally unmap here, but instead just mark that it could be |
| 455 | // unmapped (if the iova refcnt drops to zero), but then later |
| 456 | // if another _get_iova_locked() fails we can start unmapping |
| 457 | // things that are no longer needed.. |
| 458 | } |
| 459 | |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 460 | void msm_gem_aspace_domain_attach_detach_update( |
| 461 | struct msm_gem_address_space *aspace, |
| 462 | bool is_detach) |
| 463 | { |
| 464 | struct msm_gem_object *msm_obj; |
| 465 | struct drm_gem_object *obj; |
| 466 | struct aspace_client *aclient; |
| 467 | int ret; |
| 468 | uint32_t iova; |
| 469 | |
| 470 | if (!aspace) |
| 471 | return; |
| 472 | |
| 473 | mutex_lock(&aspace->dev->struct_mutex); |
| 474 | if (is_detach) { |
| 475 | /* Indicate to clients domain is getting detached */ |
| 476 | list_for_each_entry(aclient, &aspace->clients, list) { |
| 477 | if (aclient->cb) |
| 478 | aclient->cb(aclient->cb_data, |
| 479 | is_detach); |
| 480 | } |
| 481 | |
| 482 | /** |
| 483 | * Unmap active buffers, |
| 484 | * typically clients should do this when the callback is called, |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 485 | * but this needs to be done for the buffers which are not |
| 486 | * attached to any planes. |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 487 | */ |
| 488 | list_for_each_entry(msm_obj, &aspace->active_list, iova_list) { |
| 489 | obj = &msm_obj->base; |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 490 | if (obj->import_attach) |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 491 | put_iova(obj); |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 492 | } |
| 493 | } else { |
| 494 | /* map active buffers */ |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 495 | list_for_each_entry(msm_obj, &aspace->active_list, iova_list) { |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 496 | obj = &msm_obj->base; |
| 497 | ret = msm_gem_get_iova_locked(obj, aspace, &iova); |
| 498 | if (ret) { |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 499 | mutex_unlock(&aspace->dev->struct_mutex); |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 500 | return; |
| 501 | } |
| 502 | } |
| 503 | |
| 504 | /* Indicate to clients domain is attached */ |
| 505 | list_for_each_entry(aclient, &aspace->clients, list) { |
| 506 | if (aclient->cb) |
| 507 | aclient->cb(aclient->cb_data, |
| 508 | is_detach); |
| 509 | } |
| 510 | } |
| 511 | mutex_unlock(&aspace->dev->struct_mutex); |
| 512 | } |
| 513 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 514 | int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev, |
| 515 | struct drm_mode_create_dumb *args) |
| 516 | { |
| 517 | args->pitch = align_pitch(args->width, args->bpp); |
| 518 | args->size = PAGE_ALIGN(args->pitch * args->height); |
| 519 | return msm_gem_new_handle(dev, file, args->size, |
| 520 | MSM_BO_SCANOUT | MSM_BO_WC, &args->handle); |
| 521 | } |
| 522 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 523 | int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, |
| 524 | uint32_t handle, uint64_t *offset) |
| 525 | { |
| 526 | struct drm_gem_object *obj; |
| 527 | int ret = 0; |
| 528 | |
| 529 | /* GEM does all our handle to object mapping */ |
Chris Wilson | a8ad0bd | 2016-05-09 11:04:54 +0100 | [diff] [blame] | 530 | obj = drm_gem_object_lookup(file, handle); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 531 | if (obj == NULL) { |
| 532 | ret = -ENOENT; |
| 533 | goto fail; |
| 534 | } |
| 535 | |
| 536 | *offset = msm_gem_mmap_offset(obj); |
| 537 | |
| 538 | drm_gem_object_unreference_unlocked(obj); |
| 539 | |
| 540 | fail: |
| 541 | return ret; |
| 542 | } |
| 543 | |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 544 | void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 545 | { |
| 546 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 547 | WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex)); |
| 548 | if (!msm_obj->vaddr) { |
| 549 | struct page **pages = get_pages(obj); |
| 550 | if (IS_ERR(pages)) |
| 551 | return ERR_CAST(pages); |
| 552 | msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT, |
| 553 | VM_MAP, pgprot_writecombine(PAGE_KERNEL)); |
Rob Clark | 69a834c | 2016-05-24 18:29:38 -0400 | [diff] [blame] | 554 | if (msm_obj->vaddr == NULL) |
| 555 | return ERR_PTR(-ENOMEM); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 556 | } |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 557 | msm_obj->vmap_count++; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 558 | return msm_obj->vaddr; |
| 559 | } |
| 560 | |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 561 | void *msm_gem_get_vaddr(struct drm_gem_object *obj) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 562 | { |
| 563 | void *ret; |
| 564 | mutex_lock(&obj->dev->struct_mutex); |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 565 | ret = msm_gem_get_vaddr_locked(obj); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 566 | mutex_unlock(&obj->dev->struct_mutex); |
| 567 | return ret; |
| 568 | } |
| 569 | |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 570 | void msm_gem_put_vaddr_locked(struct drm_gem_object *obj) |
| 571 | { |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 572 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 573 | WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex)); |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 574 | WARN_ON(msm_obj->vmap_count < 1); |
| 575 | msm_obj->vmap_count--; |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 576 | } |
| 577 | |
| 578 | void msm_gem_put_vaddr(struct drm_gem_object *obj) |
| 579 | { |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 580 | mutex_lock(&obj->dev->struct_mutex); |
| 581 | msm_gem_put_vaddr_locked(obj); |
| 582 | mutex_unlock(&obj->dev->struct_mutex); |
Rob Clark | 18f2304 | 2016-05-26 16:24:35 -0400 | [diff] [blame] | 583 | } |
| 584 | |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 585 | /* Update madvise status, returns true if not purged, else |
| 586 | * false or -errno. |
| 587 | */ |
| 588 | int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv) |
| 589 | { |
| 590 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 591 | |
| 592 | WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex)); |
| 593 | |
| 594 | if (msm_obj->madv != __MSM_MADV_PURGED) |
| 595 | msm_obj->madv = madv; |
| 596 | |
| 597 | return (msm_obj->madv != __MSM_MADV_PURGED); |
| 598 | } |
| 599 | |
Rob Clark | 6820939 | 2016-05-17 16:19:32 -0400 | [diff] [blame] | 600 | void msm_gem_purge(struct drm_gem_object *obj) |
| 601 | { |
| 602 | struct drm_device *dev = obj->dev; |
| 603 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 604 | |
| 605 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 606 | WARN_ON(!is_purgeable(msm_obj)); |
| 607 | WARN_ON(obj->import_attach); |
| 608 | |
| 609 | put_iova(obj); |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 610 | msm_gem_remove_obj_from_aspace_active_list(msm_obj->aspace, obj); |
Rob Clark | 6820939 | 2016-05-17 16:19:32 -0400 | [diff] [blame] | 611 | |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 612 | msm_gem_vunmap(obj); |
Rob Clark | 6820939 | 2016-05-17 16:19:32 -0400 | [diff] [blame] | 613 | |
| 614 | put_pages(obj); |
| 615 | |
| 616 | msm_obj->madv = __MSM_MADV_PURGED; |
| 617 | |
| 618 | drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); |
| 619 | drm_gem_free_mmap_offset(obj); |
| 620 | |
| 621 | /* Our goal here is to return as much of the memory as |
| 622 | * is possible back to the system as we are called from OOM. |
| 623 | * To do this we must instruct the shmfs to drop all of its |
| 624 | * backing pages, *now*. |
| 625 | */ |
| 626 | shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1); |
| 627 | |
| 628 | invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, |
| 629 | 0, (loff_t)-1); |
| 630 | } |
| 631 | |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 632 | void msm_gem_vunmap(struct drm_gem_object *obj) |
| 633 | { |
| 634 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 635 | |
| 636 | if (!msm_obj->vaddr || WARN_ON(!is_vunmapable(msm_obj))) |
| 637 | return; |
| 638 | |
| 639 | vunmap(msm_obj->vaddr); |
| 640 | msm_obj->vaddr = NULL; |
| 641 | } |
| 642 | |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 643 | /* must be called before _move_to_active().. */ |
| 644 | int msm_gem_sync_object(struct drm_gem_object *obj, |
| 645 | struct msm_fence_context *fctx, bool exclusive) |
| 646 | { |
| 647 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 648 | struct reservation_object_list *fobj; |
| 649 | struct fence *fence; |
| 650 | int i, ret; |
| 651 | |
| 652 | if (!exclusive) { |
| 653 | /* NOTE: _reserve_shared() must happen before _add_shared_fence(), |
| 654 | * which makes this a slightly strange place to call it. OTOH this |
| 655 | * is a convenient can-fail point to hook it in. (And similar to |
| 656 | * how etnaviv and nouveau handle this.) |
| 657 | */ |
| 658 | ret = reservation_object_reserve_shared(msm_obj->resv); |
| 659 | if (ret) |
| 660 | return ret; |
| 661 | } |
| 662 | |
| 663 | fobj = reservation_object_get_list(msm_obj->resv); |
| 664 | if (!fobj || (fobj->shared_count == 0)) { |
| 665 | fence = reservation_object_get_excl(msm_obj->resv); |
| 666 | /* don't need to wait on our own fences, since ring is fifo */ |
| 667 | if (fence && (fence->context != fctx->context)) { |
| 668 | ret = fence_wait(fence, true); |
| 669 | if (ret) |
| 670 | return ret; |
| 671 | } |
| 672 | } |
| 673 | |
| 674 | if (!exclusive || !fobj) |
| 675 | return 0; |
| 676 | |
| 677 | for (i = 0; i < fobj->shared_count; i++) { |
| 678 | fence = rcu_dereference_protected(fobj->shared[i], |
| 679 | reservation_object_held(msm_obj->resv)); |
| 680 | if (fence->context != fctx->context) { |
| 681 | ret = fence_wait(fence, true); |
| 682 | if (ret) |
| 683 | return ret; |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | return 0; |
| 688 | } |
| 689 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 690 | void msm_gem_move_to_active(struct drm_gem_object *obj, |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 691 | struct msm_gpu *gpu, bool exclusive, struct fence *fence) |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 692 | { |
| 693 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 694 | WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED); |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 695 | msm_obj->gpu = gpu; |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 696 | if (exclusive) |
| 697 | reservation_object_add_excl_fence(msm_obj->resv, fence); |
Rob Clark | bf6811f | 2013-09-01 13:25:09 -0400 | [diff] [blame] | 698 | else |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 699 | reservation_object_add_shared_fence(msm_obj->resv, fence); |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 700 | list_del_init(&msm_obj->mm_list); |
| 701 | list_add_tail(&msm_obj->mm_list, &gpu->active_list); |
| 702 | } |
| 703 | |
| 704 | void msm_gem_move_to_inactive(struct drm_gem_object *obj) |
| 705 | { |
| 706 | struct drm_device *dev = obj->dev; |
| 707 | struct msm_drm_private *priv = dev->dev_private; |
| 708 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 709 | |
| 710 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 711 | |
| 712 | msm_obj->gpu = NULL; |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 713 | list_del_init(&msm_obj->mm_list); |
| 714 | list_add_tail(&msm_obj->mm_list, &priv->inactive_list); |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 715 | } |
| 716 | |
Rob Clark | ba00c3f | 2016-03-16 18:18:17 -0400 | [diff] [blame] | 717 | int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout) |
| 718 | { |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 719 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
| 720 | bool write = !!(op & MSM_PREP_WRITE); |
Chris Wilson | f755e22 | 2016-08-29 08:08:26 +0100 | [diff] [blame] | 721 | unsigned long remain = |
| 722 | op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout); |
| 723 | long ret; |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 724 | |
Chris Wilson | f755e22 | 2016-08-29 08:08:26 +0100 | [diff] [blame] | 725 | ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write, |
| 726 | true, remain); |
| 727 | if (ret == 0) |
| 728 | return remain == 0 ? -EBUSY : -ETIMEDOUT; |
| 729 | else if (ret < 0) |
| 730 | return ret; |
Rob Clark | ba00c3f | 2016-03-16 18:18:17 -0400 | [diff] [blame] | 731 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 732 | /* TODO cache maintenance */ |
| 733 | |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 734 | return 0; |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 735 | } |
| 736 | |
| 737 | int msm_gem_cpu_fini(struct drm_gem_object *obj) |
| 738 | { |
| 739 | /* TODO cache maintenance */ |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 740 | return 0; |
| 741 | } |
| 742 | |
| 743 | #ifdef CONFIG_DEBUG_FS |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 744 | static void describe_fence(struct fence *fence, const char *type, |
| 745 | struct seq_file *m) |
| 746 | { |
| 747 | if (!fence_is_signaled(fence)) |
| 748 | seq_printf(m, "\t%9s: %s %s seq %u\n", type, |
| 749 | fence->ops->get_driver_name(fence), |
| 750 | fence->ops->get_timeline_name(fence), |
| 751 | fence->seqno); |
| 752 | } |
| 753 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 754 | void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m) |
| 755 | { |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 756 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 757 | struct msm_gem_vma *domain; |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 758 | struct reservation_object *robj = msm_obj->resv; |
| 759 | struct reservation_object_list *fobj; |
| 760 | struct fence *fence; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 761 | uint64_t off = drm_vma_node_start(&obj->vma_node); |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 762 | const char *madv; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 763 | |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 764 | WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex)); |
| 765 | |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 766 | switch (msm_obj->madv) { |
| 767 | case __MSM_MADV_PURGED: |
| 768 | madv = " purged"; |
| 769 | break; |
| 770 | case MSM_MADV_DONTNEED: |
| 771 | madv = " purgeable"; |
| 772 | break; |
| 773 | case MSM_MADV_WILLNEED: |
| 774 | default: |
| 775 | madv = ""; |
| 776 | break; |
| 777 | } |
| 778 | |
| 779 | seq_printf(m, "%08x: %c %2d (%2d) %08llx %p %zu%s\n", |
Rob Clark | e22a2fb | 2017-02-13 10:14:11 -0700 | [diff] [blame] | 780 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 781 | msm_obj->flags, is_active(msm_obj) ? 'A' : 'I', |
Rob Clark | bf6811f | 2013-09-01 13:25:09 -0400 | [diff] [blame] | 782 | obj->name, obj->refcount.refcount.counter, |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 783 | off, msm_obj->vaddr, obj->size, madv); |
Rob Clark | b6295f9 | 2016-03-15 18:26:28 -0400 | [diff] [blame] | 784 | |
| 785 | rcu_read_lock(); |
| 786 | fobj = rcu_dereference(robj->fence); |
| 787 | if (fobj) { |
| 788 | unsigned int i, shared_count = fobj->shared_count; |
| 789 | |
| 790 | for (i = 0; i < shared_count; i++) { |
| 791 | fence = rcu_dereference(fobj->shared[i]); |
| 792 | describe_fence(fence, "Shared", m); |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | fence = rcu_dereference(robj->fence_excl); |
| 797 | if (fence) |
| 798 | describe_fence(fence, "Exclusive", m); |
| 799 | rcu_read_unlock(); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 800 | |
| 801 | /* FIXME: we need to print the address space here too */ |
| 802 | list_for_each_entry(domain, &msm_obj->domains, list) |
| 803 | seq_printf(m, " %08llx", domain->iova); |
| 804 | |
| 805 | seq_puts(m, "\n"); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 806 | } |
| 807 | |
| 808 | void msm_gem_describe_objects(struct list_head *list, struct seq_file *m) |
| 809 | { |
| 810 | struct msm_gem_object *msm_obj; |
| 811 | int count = 0; |
| 812 | size_t size = 0; |
| 813 | |
| 814 | list_for_each_entry(msm_obj, list, mm_list) { |
| 815 | struct drm_gem_object *obj = &msm_obj->base; |
| 816 | seq_printf(m, " "); |
| 817 | msm_gem_describe(obj, m); |
| 818 | count++; |
| 819 | size += obj->size; |
| 820 | } |
| 821 | |
| 822 | seq_printf(m, "Total %d objects, %zu bytes\n", count, size); |
| 823 | } |
| 824 | #endif |
| 825 | |
| 826 | void msm_gem_free_object(struct drm_gem_object *obj) |
| 827 | { |
| 828 | struct drm_device *dev = obj->dev; |
| 829 | struct msm_gem_object *msm_obj = to_msm_bo(obj); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 830 | |
| 831 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 832 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 833 | /* object should not be on active list: */ |
| 834 | WARN_ON(is_active(msm_obj)); |
| 835 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 836 | list_del(&msm_obj->mm_list); |
| 837 | |
Rob Clark | 4fe5f65 | 2016-06-01 11:38:28 -0400 | [diff] [blame] | 838 | put_iova(obj); |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 839 | msm_gem_remove_obj_from_aspace_active_list(msm_obj->aspace, obj); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 840 | |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 841 | if (obj->import_attach) { |
| 842 | if (msm_obj->vaddr) |
Jordan Crouse | 12bf362 | 2017-02-13 10:14:11 -0700 | [diff] [blame] | 843 | dma_buf_vunmap(obj->import_attach->dmabuf, |
| 844 | msm_obj->vaddr); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 845 | |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 846 | /* Don't drop the pages for imported dmabuf, as they are not |
| 847 | * ours, just free the array we allocated: |
| 848 | */ |
| 849 | if (msm_obj->pages) |
| 850 | drm_free_large(msm_obj->pages); |
| 851 | |
jilai wang | f28730c | 2015-04-07 13:51:32 -0400 | [diff] [blame] | 852 | drm_prime_gem_destroy(obj, msm_obj->sgt); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 853 | } else { |
Rob Clark | e1e9db2 | 2016-05-27 11:16:28 -0400 | [diff] [blame] | 854 | msm_gem_vunmap(obj); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 855 | put_pages(obj); |
| 856 | } |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 857 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 858 | if (msm_obj->resv == &msm_obj->_resv) |
| 859 | reservation_object_fini(msm_obj->resv); |
| 860 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 861 | drm_gem_object_release(obj); |
| 862 | |
| 863 | kfree(msm_obj); |
| 864 | } |
| 865 | |
| 866 | /* convenience method to construct a GEM buffer object, and userspace handle */ |
| 867 | int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, |
| 868 | uint32_t size, uint32_t flags, uint32_t *handle) |
| 869 | { |
| 870 | struct drm_gem_object *obj; |
| 871 | int ret; |
| 872 | |
| 873 | ret = mutex_lock_interruptible(&dev->struct_mutex); |
| 874 | if (ret) |
| 875 | return ret; |
| 876 | |
| 877 | obj = msm_gem_new(dev, size, flags); |
| 878 | |
| 879 | mutex_unlock(&dev->struct_mutex); |
| 880 | |
| 881 | if (IS_ERR(obj)) |
| 882 | return PTR_ERR(obj); |
| 883 | |
| 884 | ret = drm_gem_handle_create(file, obj, handle); |
| 885 | |
| 886 | /* drop reference from allocate - handle holds it now */ |
| 887 | drm_gem_object_unreference_unlocked(obj); |
| 888 | |
| 889 | return ret; |
| 890 | } |
| 891 | |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 892 | static int msm_gem_new_impl(struct drm_device *dev, |
| 893 | uint32_t size, uint32_t flags, |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 894 | struct reservation_object *resv, |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 895 | struct drm_gem_object **obj) |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 896 | { |
| 897 | struct msm_drm_private *priv = dev->dev_private; |
| 898 | struct msm_gem_object *msm_obj; |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 899 | bool use_vram = false; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 900 | |
| 901 | switch (flags & MSM_BO_CACHE_MASK) { |
| 902 | case MSM_BO_UNCACHED: |
| 903 | case MSM_BO_CACHED: |
| 904 | case MSM_BO_WC: |
| 905 | break; |
| 906 | default: |
| 907 | dev_err(dev->dev, "invalid cache flag: %x\n", |
| 908 | (flags & MSM_BO_CACHE_MASK)); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 909 | return -EINVAL; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 910 | } |
| 911 | |
Lloyd Atkinson | c30fdd7 | 2017-09-13 09:21:06 -0400 | [diff] [blame] | 912 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 913 | |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 914 | if (!iommu_present(&platform_bus_type)) |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 915 | use_vram = true; |
| 916 | else if ((flags & MSM_BO_STOLEN) && priv->vram.size) |
| 917 | use_vram = true; |
| 918 | |
| 919 | if (WARN_ON(use_vram && !priv->vram.size)) |
| 920 | return -EINVAL; |
| 921 | |
Rob Clark | e22a2fb | 2017-02-13 10:14:11 -0700 | [diff] [blame] | 922 | msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 923 | if (!msm_obj) |
| 924 | return -ENOMEM; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 925 | |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 926 | if (use_vram) { |
| 927 | struct msm_gem_vma *domain = obj_add_domain(&msm_obj->base, |
| 928 | NULL); |
| 929 | |
| 930 | if (!IS_ERR(domain)) |
| 931 | msm_obj->vram_node = &domain->node; |
| 932 | } |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 933 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 934 | msm_obj->flags = flags; |
Rob Clark | 4cd33c4 | 2016-05-17 15:44:49 -0400 | [diff] [blame] | 935 | msm_obj->madv = MSM_MADV_WILLNEED; |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 936 | |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 937 | if (resv) { |
| 938 | msm_obj->resv = resv; |
| 939 | } else { |
| 940 | msm_obj->resv = &msm_obj->_resv; |
| 941 | reservation_object_init(msm_obj->resv); |
| 942 | } |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 943 | |
Rob Clark | 7198e6b | 2013-07-19 12:59:32 -0400 | [diff] [blame] | 944 | INIT_LIST_HEAD(&msm_obj->submit_entry); |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 945 | INIT_LIST_HEAD(&msm_obj->domains); |
Abhijit Kulkarni | f4657b1 | 2017-06-28 18:40:19 -0700 | [diff] [blame] | 946 | INIT_LIST_HEAD(&msm_obj->iova_list); |
Abhijit Kulkarni | 75c31e7 | 2017-07-25 17:31:33 -0700 | [diff] [blame] | 947 | msm_obj->aspace = NULL; |
Jordan Crouse | d8e9652 | 2017-02-13 10:14:16 -0700 | [diff] [blame] | 948 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 949 | list_add_tail(&msm_obj->mm_list, &priv->inactive_list); |
| 950 | |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 951 | *obj = &msm_obj->base; |
| 952 | |
| 953 | return 0; |
| 954 | } |
| 955 | |
| 956 | struct drm_gem_object *msm_gem_new(struct drm_device *dev, |
| 957 | uint32_t size, uint32_t flags) |
| 958 | { |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 959 | struct drm_gem_object *obj = NULL; |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 960 | int ret; |
| 961 | |
| 962 | WARN_ON(!mutex_is_locked(&dev->struct_mutex)); |
| 963 | |
| 964 | size = PAGE_ALIGN(size); |
| 965 | |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 966 | ret = msm_gem_new_impl(dev, size, flags, NULL, &obj); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 967 | if (ret) |
| 968 | goto fail; |
| 969 | |
Rob Clark | 072f1f9 | 2015-03-03 15:04:25 -0500 | [diff] [blame] | 970 | if (use_pages(obj)) { |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 971 | ret = drm_gem_object_init(dev, obj, size); |
| 972 | if (ret) |
| 973 | goto fail; |
| 974 | } else { |
| 975 | drm_gem_private_object_init(dev, obj, size); |
| 976 | } |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 977 | |
| 978 | return obj; |
| 979 | |
| 980 | fail: |
Markus Elfring | 0a67712 | 2016-07-13 19:29:19 +0200 | [diff] [blame] | 981 | drm_gem_object_unreference(obj); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 982 | return ERR_PTR(ret); |
| 983 | } |
| 984 | |
| 985 | struct drm_gem_object *msm_gem_import(struct drm_device *dev, |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 986 | struct dma_buf *dmabuf, struct sg_table *sgt) |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 987 | { |
| 988 | struct msm_gem_object *msm_obj; |
Lloyd Atkinson | c30fdd7 | 2017-09-13 09:21:06 -0400 | [diff] [blame] | 989 | struct drm_gem_object *obj = NULL; |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 990 | uint32_t size; |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 991 | int ret, npages; |
| 992 | |
Rob Clark | 871d812 | 2013-11-16 12:56:06 -0500 | [diff] [blame] | 993 | /* if we don't have IOMMU, don't bother pretending we can import: */ |
| 994 | if (!iommu_present(&platform_bus_type)) { |
| 995 | dev_err(dev->dev, "cannot import without IOMMU\n"); |
| 996 | return ERR_PTR(-EINVAL); |
| 997 | } |
| 998 | |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 999 | size = PAGE_ALIGN(dmabuf->size); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 1000 | |
Lloyd Atkinson | c30fdd7 | 2017-09-13 09:21:06 -0400 | [diff] [blame] | 1001 | ret = mutex_lock_interruptible(&dev->struct_mutex); |
| 1002 | if (ret) |
| 1003 | return ERR_PTR(ret); |
| 1004 | |
Rob Clark | 79f0e20 | 2016-03-16 12:40:35 -0400 | [diff] [blame] | 1005 | ret = msm_gem_new_impl(dev, size, MSM_BO_WC, dmabuf->resv, &obj); |
Lloyd Atkinson | c30fdd7 | 2017-09-13 09:21:06 -0400 | [diff] [blame] | 1006 | mutex_unlock(&dev->struct_mutex); |
Rob Clark | 05b8491 | 2013-09-28 11:28:35 -0400 | [diff] [blame] | 1007 | if (ret) |
| 1008 | goto fail; |
| 1009 | |
| 1010 | drm_gem_private_object_init(dev, obj, size); |
| 1011 | |
| 1012 | npages = size / PAGE_SIZE; |
| 1013 | |
| 1014 | msm_obj = to_msm_bo(obj); |
| 1015 | msm_obj->sgt = sgt; |
| 1016 | msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *)); |
| 1017 | if (!msm_obj->pages) { |
| 1018 | ret = -ENOMEM; |
| 1019 | goto fail; |
| 1020 | } |
| 1021 | |
| 1022 | ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages); |
| 1023 | if (ret) |
| 1024 | goto fail; |
| 1025 | |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 1026 | return obj; |
| 1027 | |
| 1028 | fail: |
Markus Elfring | e73a856 | 2016-07-13 19:15:35 +0200 | [diff] [blame] | 1029 | drm_gem_object_unreference_unlocked(obj); |
Rob Clark | c8afe68 | 2013-06-26 12:44:06 -0400 | [diff] [blame] | 1030 | return ERR_PTR(ret); |
| 1031 | } |