Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2008 Advanced Micro Devices, Inc. |
| 3 | * Copyright 2008 Red Hat Inc. |
| 4 | * Copyright 2009 Jerome Glisse. |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 7 | * copy of this software and associated documentation files (the "Software"), |
| 8 | * to deal in the Software without restriction, including without limitation |
| 9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 10 | * and/or sell copies of the Software, and to permit persons to whom the |
| 11 | * Software is furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in |
| 14 | * all copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 22 | * OTHER DEALINGS IN THE SOFTWARE. |
| 23 | * |
| 24 | * Authors: Dave Airlie |
| 25 | * Alex Deucher |
| 26 | * Jerome Glisse |
| 27 | */ |
| 28 | #include <drm/drmP.h> |
| 29 | #include <drm/amdgpu_drm.h> |
| 30 | #include "amdgpu.h" |
| 31 | #include "amdgpu_trace.h" |
| 32 | |
| 33 | /* |
| 34 | * GPUVM |
| 35 | * GPUVM is similar to the legacy gart on older asics, however |
| 36 | * rather than there being a single global gart table |
| 37 | * for the entire GPU, there are multiple VM page tables active |
| 38 | * at any given time. The VM page tables can contain a mix |
| 39 | * vram pages and system memory pages and system memory pages |
| 40 | * can be mapped as snooped (cached system pages) or unsnooped |
| 41 | * (uncached system pages). |
| 42 | * Each VM has an ID associated with it and there is a page table |
| 43 | * associated with each VMID. When execting a command buffer, |
| 44 | * the kernel tells the the ring what VMID to use for that command |
| 45 | * buffer. VMIDs are allocated dynamically as commands are submitted. |
| 46 | * The userspace drivers maintain their own address space and the kernel |
| 47 | * sets up their pages tables accordingly when they submit their |
| 48 | * command buffers and a VMID is assigned. |
| 49 | * Cayman/Trinity support up to 8 active VMs at any given time; |
| 50 | * SI supports 16. |
| 51 | */ |
| 52 | |
| 53 | /** |
| 54 | * amdgpu_vm_num_pde - return the number of page directory entries |
| 55 | * |
| 56 | * @adev: amdgpu_device pointer |
| 57 | * |
| 58 | * Calculate the number of page directory entries (cayman+). |
| 59 | */ |
| 60 | static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev) |
| 61 | { |
| 62 | return adev->vm_manager.max_pfn >> amdgpu_vm_block_size; |
| 63 | } |
| 64 | |
| 65 | /** |
| 66 | * amdgpu_vm_directory_size - returns the size of the page directory in bytes |
| 67 | * |
| 68 | * @adev: amdgpu_device pointer |
| 69 | * |
| 70 | * Calculate the size of the page directory in bytes (cayman+). |
| 71 | */ |
| 72 | static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev) |
| 73 | { |
| 74 | return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8); |
| 75 | } |
| 76 | |
| 77 | /** |
| 78 | * amdgpu_vm_get_bos - add the vm BOs to a validation list |
| 79 | * |
| 80 | * @vm: vm providing the BOs |
| 81 | * @head: head of validation list |
| 82 | * |
| 83 | * Add the page directory to the list of BOs to |
| 84 | * validate for command submission (cayman+). |
| 85 | */ |
| 86 | struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, |
| 87 | struct amdgpu_vm *vm, |
| 88 | struct list_head *head) |
| 89 | { |
| 90 | struct amdgpu_bo_list_entry *list; |
| 91 | unsigned i, idx; |
| 92 | |
| 93 | list = drm_malloc_ab(vm->max_pde_used + 2, |
| 94 | sizeof(struct amdgpu_bo_list_entry)); |
monk.liu | 3d5a08c | 2015-05-26 10:22:41 +0800 | [diff] [blame] | 95 | if (!list) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 96 | return NULL; |
monk.liu | 3d5a08c | 2015-05-26 10:22:41 +0800 | [diff] [blame] | 97 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 98 | |
| 99 | /* add the vm page table to the list */ |
| 100 | list[0].robj = vm->page_directory; |
| 101 | list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; |
| 102 | list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; |
| 103 | list[0].priority = 0; |
| 104 | list[0].tv.bo = &vm->page_directory->tbo; |
| 105 | list[0].tv.shared = true; |
| 106 | list_add(&list[0].tv.head, head); |
| 107 | |
| 108 | for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { |
| 109 | if (!vm->page_tables[i].bo) |
| 110 | continue; |
| 111 | |
| 112 | list[idx].robj = vm->page_tables[i].bo; |
| 113 | list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; |
| 114 | list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; |
| 115 | list[idx].priority = 0; |
| 116 | list[idx].tv.bo = &list[idx].robj->tbo; |
| 117 | list[idx].tv.shared = true; |
| 118 | list_add(&list[idx++].tv.head, head); |
| 119 | } |
| 120 | |
| 121 | return list; |
| 122 | } |
| 123 | |
| 124 | /** |
| 125 | * amdgpu_vm_grab_id - allocate the next free VMID |
| 126 | * |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 127 | * @vm: vm to allocate id for |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 128 | * @ring: ring we want to submit job to |
| 129 | * @sync: sync object where we add dependencies |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 130 | * |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 131 | * Allocate an id for the vm, adding fences to the sync obj as necessary. |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 132 | * |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 133 | * Global mutex must be locked! |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 134 | */ |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 135 | int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, |
| 136 | struct amdgpu_sync *sync) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 137 | { |
Christian König | d528329 | 2015-10-22 11:55:58 +0200 | [diff] [blame] | 138 | struct fence *best[AMDGPU_MAX_RINGS] = {}; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 139 | struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; |
| 140 | struct amdgpu_device *adev = ring->adev; |
| 141 | |
| 142 | unsigned choices[2] = {}; |
| 143 | unsigned i; |
| 144 | |
| 145 | /* check if the id is still valid */ |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 146 | if (vm_id->id) { |
| 147 | unsigned id = vm_id->id; |
| 148 | long owner; |
| 149 | |
| 150 | owner = atomic_long_read(&adev->vm_manager.ids[id].owner); |
| 151 | if (owner == (long)vm) { |
| 152 | trace_amdgpu_vm_grab_id(vm_id->id, ring->idx); |
| 153 | return 0; |
| 154 | } |
Christian König | 39ff844 | 2015-09-28 12:01:20 +0200 | [diff] [blame] | 155 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 156 | |
| 157 | /* we definately need to flush */ |
| 158 | vm_id->pd_gpu_addr = ~0ll; |
| 159 | |
| 160 | /* skip over VMID 0, since it is the system VM */ |
| 161 | for (i = 1; i < adev->vm_manager.nvm; ++i) { |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 162 | struct fence *fence = adev->vm_manager.ids[i].active; |
Christian König | d528329 | 2015-10-22 11:55:58 +0200 | [diff] [blame] | 163 | struct amdgpu_ring *fring; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 164 | |
| 165 | if (fence == NULL) { |
| 166 | /* found a free one */ |
| 167 | vm_id->id = i; |
| 168 | trace_amdgpu_vm_grab_id(i, ring->idx); |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 169 | return 0; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 170 | } |
| 171 | |
Christian König | d528329 | 2015-10-22 11:55:58 +0200 | [diff] [blame] | 172 | fring = amdgpu_ring_from_fence(fence); |
| 173 | if (best[fring->idx] == NULL || |
| 174 | fence_is_later(best[fring->idx], fence)) { |
| 175 | best[fring->idx] = fence; |
| 176 | choices[fring == ring ? 0 : 1] = i; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 177 | } |
| 178 | } |
| 179 | |
| 180 | for (i = 0; i < 2; ++i) { |
| 181 | if (choices[i]) { |
Christian König | d528329 | 2015-10-22 11:55:58 +0200 | [diff] [blame] | 182 | struct fence *fence; |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 183 | |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 184 | fence = adev->vm_manager.ids[choices[i]].active; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 185 | vm_id->id = choices[i]; |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 186 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 187 | trace_amdgpu_vm_grab_id(choices[i], ring->idx); |
Christian König | d528329 | 2015-10-22 11:55:58 +0200 | [diff] [blame] | 188 | return amdgpu_sync_fence(ring->adev, sync, fence); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 189 | } |
| 190 | } |
| 191 | |
| 192 | /* should never happen */ |
| 193 | BUG(); |
Christian König | 7f8a529 | 2015-07-20 16:09:40 +0200 | [diff] [blame] | 194 | return -EINVAL; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 195 | } |
| 196 | |
| 197 | /** |
| 198 | * amdgpu_vm_flush - hardware flush the vm |
| 199 | * |
| 200 | * @ring: ring to use for flush |
| 201 | * @vm: vm we want to flush |
| 202 | * @updates: last vm update that we waited for |
| 203 | * |
| 204 | * Flush the vm (cayman+). |
| 205 | * |
| 206 | * Global and local mutex must be locked! |
| 207 | */ |
| 208 | void amdgpu_vm_flush(struct amdgpu_ring *ring, |
| 209 | struct amdgpu_vm *vm, |
Chunming Zhou | 3c62338 | 2015-08-20 18:33:59 +0800 | [diff] [blame] | 210 | struct fence *updates) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 211 | { |
| 212 | uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); |
| 213 | struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; |
Chunming Zhou | 3c62338 | 2015-08-20 18:33:59 +0800 | [diff] [blame] | 214 | struct fence *flushed_updates = vm_id->flushed_updates; |
Christian König | b56c228 | 2015-10-29 17:01:19 +0100 | [diff] [blame] | 215 | bool is_later; |
Chunming Zhou | 3c62338 | 2015-08-20 18:33:59 +0800 | [diff] [blame] | 216 | |
Christian König | b56c228 | 2015-10-29 17:01:19 +0100 | [diff] [blame] | 217 | if (!flushed_updates) |
| 218 | is_later = true; |
| 219 | else if (!updates) |
| 220 | is_later = false; |
| 221 | else |
| 222 | is_later = fence_is_later(updates, flushed_updates); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 223 | |
Christian König | b56c228 | 2015-10-29 17:01:19 +0100 | [diff] [blame] | 224 | if (pd_addr != vm_id->pd_gpu_addr || is_later) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 225 | trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); |
Christian König | b56c228 | 2015-10-29 17:01:19 +0100 | [diff] [blame] | 226 | if (is_later) { |
Chunming Zhou | 3c62338 | 2015-08-20 18:33:59 +0800 | [diff] [blame] | 227 | vm_id->flushed_updates = fence_get(updates); |
| 228 | fence_put(flushed_updates); |
| 229 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 230 | vm_id->pd_gpu_addr = pd_addr; |
| 231 | amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | /** |
| 236 | * amdgpu_vm_fence - remember fence for vm |
| 237 | * |
| 238 | * @adev: amdgpu_device pointer |
| 239 | * @vm: vm we want to fence |
| 240 | * @fence: fence to remember |
| 241 | * |
| 242 | * Fence the vm (cayman+). |
| 243 | * Set the fence used to protect page table and id. |
| 244 | * |
| 245 | * Global and local mutex must be locked! |
| 246 | */ |
| 247 | void amdgpu_vm_fence(struct amdgpu_device *adev, |
| 248 | struct amdgpu_vm *vm, |
Christian König | 16ae42f | 2015-11-03 14:53:28 +0100 | [diff] [blame] | 249 | struct fence *fence) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 250 | { |
Christian König | 16ae42f | 2015-11-03 14:53:28 +0100 | [diff] [blame] | 251 | struct amdgpu_ring *ring = amdgpu_ring_from_fence(fence); |
| 252 | unsigned vm_id = vm->ids[ring->idx].id; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 253 | |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 254 | fence_put(adev->vm_manager.ids[vm_id].active); |
| 255 | adev->vm_manager.ids[vm_id].active = fence_get(fence); |
| 256 | atomic_long_set(&adev->vm_manager.ids[vm_id].owner, (long)vm); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 257 | } |
| 258 | |
| 259 | /** |
| 260 | * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo |
| 261 | * |
| 262 | * @vm: requested vm |
| 263 | * @bo: requested buffer object |
| 264 | * |
| 265 | * Find @bo inside the requested vm (cayman+). |
| 266 | * Search inside the @bos vm list for the requested vm |
| 267 | * Returns the found bo_va or NULL if none is found |
| 268 | * |
| 269 | * Object has to be reserved! |
| 270 | */ |
| 271 | struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, |
| 272 | struct amdgpu_bo *bo) |
| 273 | { |
| 274 | struct amdgpu_bo_va *bo_va; |
| 275 | |
| 276 | list_for_each_entry(bo_va, &bo->va, bo_list) { |
| 277 | if (bo_va->vm == vm) { |
| 278 | return bo_va; |
| 279 | } |
| 280 | } |
| 281 | return NULL; |
| 282 | } |
| 283 | |
| 284 | /** |
| 285 | * amdgpu_vm_update_pages - helper to call the right asic function |
| 286 | * |
| 287 | * @adev: amdgpu_device pointer |
| 288 | * @ib: indirect buffer to fill with commands |
| 289 | * @pe: addr of the page entry |
| 290 | * @addr: dst addr to write into pe |
| 291 | * @count: number of page entries to update |
| 292 | * @incr: increase next addr by incr bytes |
| 293 | * @flags: hw access flags |
| 294 | * @gtt_flags: GTT hw access flags |
| 295 | * |
| 296 | * Traces the parameters and calls the right asic functions |
| 297 | * to setup the page table using the DMA. |
| 298 | */ |
| 299 | static void amdgpu_vm_update_pages(struct amdgpu_device *adev, |
| 300 | struct amdgpu_ib *ib, |
| 301 | uint64_t pe, uint64_t addr, |
| 302 | unsigned count, uint32_t incr, |
| 303 | uint32_t flags, uint32_t gtt_flags) |
| 304 | { |
| 305 | trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); |
| 306 | |
| 307 | if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { |
| 308 | uint64_t src = adev->gart.table_addr + (addr >> 12) * 8; |
| 309 | amdgpu_vm_copy_pte(adev, ib, pe, src, count); |
| 310 | |
| 311 | } else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) { |
| 312 | amdgpu_vm_write_pte(adev, ib, pe, addr, |
| 313 | count, incr, flags); |
| 314 | |
| 315 | } else { |
| 316 | amdgpu_vm_set_pte_pde(adev, ib, pe, addr, |
| 317 | count, incr, flags); |
| 318 | } |
| 319 | } |
| 320 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 321 | int amdgpu_vm_free_job(struct amdgpu_job *job) |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 322 | { |
| 323 | int i; |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 324 | for (i = 0; i < job->num_ibs; i++) |
| 325 | amdgpu_ib_free(job->adev, &job->ibs[i]); |
| 326 | kfree(job->ibs); |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 327 | return 0; |
| 328 | } |
| 329 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 330 | /** |
| 331 | * amdgpu_vm_clear_bo - initially clear the page dir/table |
| 332 | * |
| 333 | * @adev: amdgpu_device pointer |
| 334 | * @bo: bo to clear |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 335 | * |
| 336 | * need to reserve bo first before calling it. |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 337 | */ |
| 338 | static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, |
| 339 | struct amdgpu_bo *bo) |
| 340 | { |
| 341 | struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 342 | struct fence *fence = NULL; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 343 | struct amdgpu_ib *ib; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 344 | unsigned entries; |
| 345 | uint64_t addr; |
| 346 | int r; |
| 347 | |
monk.liu | ca95261 | 2015-05-25 14:44:05 +0800 | [diff] [blame] | 348 | r = reservation_object_reserve_shared(bo->tbo.resv); |
| 349 | if (r) |
| 350 | return r; |
| 351 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 352 | r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); |
| 353 | if (r) |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 354 | goto error; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 355 | |
| 356 | addr = amdgpu_bo_gpu_offset(bo); |
| 357 | entries = amdgpu_bo_size(bo) / 8; |
| 358 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 359 | ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); |
| 360 | if (!ib) |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 361 | goto error; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 362 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 363 | r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 364 | if (r) |
| 365 | goto error_free; |
| 366 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 367 | ib->length_dw = 0; |
| 368 | |
| 369 | amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); |
| 370 | amdgpu_vm_pad_ib(adev, ib); |
| 371 | WARN_ON(ib->length_dw > 64); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 372 | r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, |
| 373 | &amdgpu_vm_free_job, |
| 374 | AMDGPU_FENCE_OWNER_VM, |
| 375 | &fence); |
| 376 | if (!r) |
| 377 | amdgpu_bo_fence(bo, fence, true); |
Chunming Zhou | 281b422 | 2015-08-12 12:58:31 +0800 | [diff] [blame] | 378 | fence_put(fence); |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 379 | if (amdgpu_enable_scheduler) |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 380 | return 0; |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 381 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 382 | error_free: |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 383 | amdgpu_ib_free(adev, ib); |
| 384 | kfree(ib); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 385 | |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 386 | error: |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 387 | return r; |
| 388 | } |
| 389 | |
| 390 | /** |
| 391 | * amdgpu_vm_map_gart - get the physical address of a gart page |
| 392 | * |
| 393 | * @adev: amdgpu_device pointer |
| 394 | * @addr: the unmapped addr |
| 395 | * |
| 396 | * Look up the physical address of the page that the pte resolves |
| 397 | * to (cayman+). |
| 398 | * Returns the physical address of the page. |
| 399 | */ |
| 400 | uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr) |
| 401 | { |
| 402 | uint64_t result; |
| 403 | |
| 404 | /* page table offset */ |
| 405 | result = adev->gart.pages_addr[addr >> PAGE_SHIFT]; |
| 406 | |
| 407 | /* in case cpu page size != gpu page size*/ |
| 408 | result |= addr & (~PAGE_MASK); |
| 409 | |
| 410 | return result; |
| 411 | } |
| 412 | |
| 413 | /** |
| 414 | * amdgpu_vm_update_pdes - make sure that page directory is valid |
| 415 | * |
| 416 | * @adev: amdgpu_device pointer |
| 417 | * @vm: requested vm |
| 418 | * @start: start of GPU address range |
| 419 | * @end: end of GPU address range |
| 420 | * |
| 421 | * Allocates new page tables if necessary |
| 422 | * and updates the page directory (cayman+). |
| 423 | * Returns 0 for success, error for failure. |
| 424 | * |
| 425 | * Global and local mutex must be locked! |
| 426 | */ |
| 427 | int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, |
| 428 | struct amdgpu_vm *vm) |
| 429 | { |
| 430 | struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; |
| 431 | struct amdgpu_bo *pd = vm->page_directory; |
| 432 | uint64_t pd_addr = amdgpu_bo_gpu_offset(pd); |
| 433 | uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; |
| 434 | uint64_t last_pde = ~0, last_pt = ~0; |
| 435 | unsigned count = 0, pt_idx, ndw; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 436 | struct amdgpu_ib *ib; |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 437 | struct fence *fence = NULL; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 438 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 439 | int r; |
| 440 | |
| 441 | /* padding, etc. */ |
| 442 | ndw = 64; |
| 443 | |
| 444 | /* assume the worst case */ |
| 445 | ndw += vm->max_pde_used * 6; |
| 446 | |
| 447 | /* update too big for an IB */ |
| 448 | if (ndw > 0xfffff) |
| 449 | return -ENOMEM; |
| 450 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 451 | ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); |
| 452 | if (!ib) |
| 453 | return -ENOMEM; |
| 454 | |
| 455 | r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); |
Sudip Mukherjee | 7a57455 | 2015-10-08 19:28:01 +0530 | [diff] [blame] | 456 | if (r) { |
| 457 | kfree(ib); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 458 | return r; |
Sudip Mukherjee | 7a57455 | 2015-10-08 19:28:01 +0530 | [diff] [blame] | 459 | } |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 460 | ib->length_dw = 0; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 461 | |
| 462 | /* walk over the address space and update the page directory */ |
| 463 | for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { |
| 464 | struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo; |
| 465 | uint64_t pde, pt; |
| 466 | |
| 467 | if (bo == NULL) |
| 468 | continue; |
| 469 | |
| 470 | pt = amdgpu_bo_gpu_offset(bo); |
| 471 | if (vm->page_tables[pt_idx].addr == pt) |
| 472 | continue; |
| 473 | vm->page_tables[pt_idx].addr = pt; |
| 474 | |
| 475 | pde = pd_addr + pt_idx * 8; |
| 476 | if (((last_pde + 8 * count) != pde) || |
| 477 | ((last_pt + incr * count) != pt)) { |
| 478 | |
| 479 | if (count) { |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 480 | amdgpu_vm_update_pages(adev, ib, last_pde, |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 481 | last_pt, count, incr, |
| 482 | AMDGPU_PTE_VALID, 0); |
| 483 | } |
| 484 | |
| 485 | count = 1; |
| 486 | last_pde = pde; |
| 487 | last_pt = pt; |
| 488 | } else { |
| 489 | ++count; |
| 490 | } |
| 491 | } |
| 492 | |
| 493 | if (count) |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 494 | amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 495 | incr, AMDGPU_PTE_VALID, 0); |
| 496 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 497 | if (ib->length_dw != 0) { |
| 498 | amdgpu_vm_pad_ib(adev, ib); |
| 499 | amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); |
| 500 | WARN_ON(ib->length_dw > ndw); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 501 | r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, |
| 502 | &amdgpu_vm_free_job, |
| 503 | AMDGPU_FENCE_OWNER_VM, |
| 504 | &fence); |
| 505 | if (r) |
| 506 | goto error_free; |
Bas Nieuwenhuizen | 05906de | 2015-08-14 20:08:40 +0200 | [diff] [blame] | 507 | |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 508 | amdgpu_bo_fence(pd, fence, true); |
Bas Nieuwenhuizen | 05906de | 2015-08-14 20:08:40 +0200 | [diff] [blame] | 509 | fence_put(vm->page_directory_fence); |
| 510 | vm->page_directory_fence = fence_get(fence); |
Chunming Zhou | 281b422 | 2015-08-12 12:58:31 +0800 | [diff] [blame] | 511 | fence_put(fence); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 512 | } |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 513 | |
| 514 | if (!amdgpu_enable_scheduler || ib->length_dw == 0) { |
| 515 | amdgpu_ib_free(adev, ib); |
| 516 | kfree(ib); |
| 517 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 518 | |
| 519 | return 0; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 520 | |
| 521 | error_free: |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 522 | amdgpu_ib_free(adev, ib); |
| 523 | kfree(ib); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 524 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 525 | } |
| 526 | |
| 527 | /** |
| 528 | * amdgpu_vm_frag_ptes - add fragment information to PTEs |
| 529 | * |
| 530 | * @adev: amdgpu_device pointer |
| 531 | * @ib: IB for the update |
| 532 | * @pe_start: first PTE to handle |
| 533 | * @pe_end: last PTE to handle |
| 534 | * @addr: addr those PTEs should point to |
| 535 | * @flags: hw mapping flags |
| 536 | * @gtt_flags: GTT hw mapping flags |
| 537 | * |
| 538 | * Global and local mutex must be locked! |
| 539 | */ |
| 540 | static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, |
| 541 | struct amdgpu_ib *ib, |
| 542 | uint64_t pe_start, uint64_t pe_end, |
| 543 | uint64_t addr, uint32_t flags, |
| 544 | uint32_t gtt_flags) |
| 545 | { |
| 546 | /** |
| 547 | * The MC L1 TLB supports variable sized pages, based on a fragment |
| 548 | * field in the PTE. When this field is set to a non-zero value, page |
| 549 | * granularity is increased from 4KB to (1 << (12 + frag)). The PTE |
| 550 | * flags are considered valid for all PTEs within the fragment range |
| 551 | * and corresponding mappings are assumed to be physically contiguous. |
| 552 | * |
| 553 | * The L1 TLB can store a single PTE for the whole fragment, |
| 554 | * significantly increasing the space available for translation |
| 555 | * caching. This leads to large improvements in throughput when the |
| 556 | * TLB is under pressure. |
| 557 | * |
| 558 | * The L2 TLB distributes small and large fragments into two |
| 559 | * asymmetric partitions. The large fragment cache is significantly |
| 560 | * larger. Thus, we try to use large fragments wherever possible. |
| 561 | * Userspace can support this by aligning virtual base address and |
| 562 | * allocation size to the fragment size. |
| 563 | */ |
| 564 | |
| 565 | /* SI and newer are optimized for 64KB */ |
| 566 | uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB; |
| 567 | uint64_t frag_align = 0x80; |
| 568 | |
| 569 | uint64_t frag_start = ALIGN(pe_start, frag_align); |
| 570 | uint64_t frag_end = pe_end & ~(frag_align - 1); |
| 571 | |
| 572 | unsigned count; |
| 573 | |
| 574 | /* system pages are non continuously */ |
| 575 | if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) || |
| 576 | (frag_start >= frag_end)) { |
| 577 | |
| 578 | count = (pe_end - pe_start) / 8; |
| 579 | amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, |
| 580 | AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); |
| 581 | return; |
| 582 | } |
| 583 | |
| 584 | /* handle the 4K area at the beginning */ |
| 585 | if (pe_start != frag_start) { |
| 586 | count = (frag_start - pe_start) / 8; |
| 587 | amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, |
| 588 | AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); |
| 589 | addr += AMDGPU_GPU_PAGE_SIZE * count; |
| 590 | } |
| 591 | |
| 592 | /* handle the area in the middle */ |
| 593 | count = (frag_end - frag_start) / 8; |
| 594 | amdgpu_vm_update_pages(adev, ib, frag_start, addr, count, |
| 595 | AMDGPU_GPU_PAGE_SIZE, flags | frag_flags, |
| 596 | gtt_flags); |
| 597 | |
| 598 | /* handle the 4K area at the end */ |
| 599 | if (frag_end != pe_end) { |
| 600 | addr += AMDGPU_GPU_PAGE_SIZE * count; |
| 601 | count = (pe_end - frag_end) / 8; |
| 602 | amdgpu_vm_update_pages(adev, ib, frag_end, addr, count, |
| 603 | AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); |
| 604 | } |
| 605 | } |
| 606 | |
| 607 | /** |
| 608 | * amdgpu_vm_update_ptes - make sure that page tables are valid |
| 609 | * |
| 610 | * @adev: amdgpu_device pointer |
| 611 | * @vm: requested vm |
| 612 | * @start: start of GPU address range |
| 613 | * @end: end of GPU address range |
| 614 | * @dst: destination address to map to |
| 615 | * @flags: mapping flags |
| 616 | * |
| 617 | * Update the page tables in the range @start - @end (cayman+). |
| 618 | * |
| 619 | * Global and local mutex must be locked! |
| 620 | */ |
| 621 | static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, |
| 622 | struct amdgpu_vm *vm, |
| 623 | struct amdgpu_ib *ib, |
| 624 | uint64_t start, uint64_t end, |
| 625 | uint64_t dst, uint32_t flags, |
| 626 | uint32_t gtt_flags) |
| 627 | { |
| 628 | uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; |
| 629 | uint64_t last_pte = ~0, last_dst = ~0; |
Christian König | a60c423 | 2015-09-01 15:33:25 +0200 | [diff] [blame] | 630 | void *owner = AMDGPU_FENCE_OWNER_VM; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 631 | unsigned count = 0; |
| 632 | uint64_t addr; |
| 633 | |
Christian König | a60c423 | 2015-09-01 15:33:25 +0200 | [diff] [blame] | 634 | /* sync to everything on unmapping */ |
| 635 | if (!(flags & AMDGPU_PTE_VALID)) |
| 636 | owner = AMDGPU_FENCE_OWNER_UNDEFINED; |
| 637 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 638 | /* walk over the address space and update the page tables */ |
| 639 | for (addr = start; addr < end; ) { |
| 640 | uint64_t pt_idx = addr >> amdgpu_vm_block_size; |
| 641 | struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo; |
| 642 | unsigned nptes; |
| 643 | uint64_t pte; |
| 644 | int r; |
| 645 | |
Christian König | a60c423 | 2015-09-01 15:33:25 +0200 | [diff] [blame] | 646 | amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 647 | r = reservation_object_reserve_shared(pt->tbo.resv); |
| 648 | if (r) |
| 649 | return r; |
| 650 | |
| 651 | if ((addr & ~mask) == (end & ~mask)) |
| 652 | nptes = end - addr; |
| 653 | else |
| 654 | nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); |
| 655 | |
| 656 | pte = amdgpu_bo_gpu_offset(pt); |
| 657 | pte += (addr & mask) * 8; |
| 658 | |
| 659 | if ((last_pte + 8 * count) != pte) { |
| 660 | |
| 661 | if (count) { |
| 662 | amdgpu_vm_frag_ptes(adev, ib, last_pte, |
| 663 | last_pte + 8 * count, |
| 664 | last_dst, flags, |
| 665 | gtt_flags); |
| 666 | } |
| 667 | |
| 668 | count = nptes; |
| 669 | last_pte = pte; |
| 670 | last_dst = dst; |
| 671 | } else { |
| 672 | count += nptes; |
| 673 | } |
| 674 | |
| 675 | addr += nptes; |
| 676 | dst += nptes * AMDGPU_GPU_PAGE_SIZE; |
| 677 | } |
| 678 | |
| 679 | if (count) { |
| 680 | amdgpu_vm_frag_ptes(adev, ib, last_pte, |
| 681 | last_pte + 8 * count, |
| 682 | last_dst, flags, gtt_flags); |
| 683 | } |
| 684 | |
| 685 | return 0; |
| 686 | } |
| 687 | |
| 688 | /** |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 689 | * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table |
| 690 | * |
| 691 | * @adev: amdgpu_device pointer |
| 692 | * @vm: requested vm |
| 693 | * @mapping: mapped range and flags to use for the update |
| 694 | * @addr: addr to set the area to |
| 695 | * @gtt_flags: flags as they are used for GTT |
| 696 | * @fence: optional resulting fence |
| 697 | * |
| 698 | * Fill in the page table entries for @mapping. |
| 699 | * Returns 0 for success, -EINVAL for failure. |
| 700 | * |
| 701 | * Object have to be reserved and mutex must be locked! |
| 702 | */ |
| 703 | static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, |
| 704 | struct amdgpu_vm *vm, |
| 705 | struct amdgpu_bo_va_mapping *mapping, |
| 706 | uint64_t addr, uint32_t gtt_flags, |
Chunming Zhou | bb1e38a4 | 2015-08-03 18:19:38 +0800 | [diff] [blame] | 707 | struct fence **fence) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 708 | { |
| 709 | struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; |
| 710 | unsigned nptes, ncmds, ndw; |
| 711 | uint32_t flags = gtt_flags; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 712 | struct amdgpu_ib *ib; |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 713 | struct fence *f = NULL; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 714 | int r; |
| 715 | |
| 716 | /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here |
| 717 | * but in case of something, we filter the flags in first place |
| 718 | */ |
| 719 | if (!(mapping->flags & AMDGPU_PTE_READABLE)) |
| 720 | flags &= ~AMDGPU_PTE_READABLE; |
| 721 | if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) |
| 722 | flags &= ~AMDGPU_PTE_WRITEABLE; |
| 723 | |
| 724 | trace_amdgpu_vm_bo_update(mapping); |
| 725 | |
| 726 | nptes = mapping->it.last - mapping->it.start + 1; |
| 727 | |
| 728 | /* |
| 729 | * reserve space for one command every (1 << BLOCK_SIZE) |
| 730 | * entries or 2k dwords (whatever is smaller) |
| 731 | */ |
| 732 | ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; |
| 733 | |
| 734 | /* padding, etc. */ |
| 735 | ndw = 64; |
| 736 | |
| 737 | if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { |
| 738 | /* only copy commands needed */ |
| 739 | ndw += ncmds * 7; |
| 740 | |
| 741 | } else if (flags & AMDGPU_PTE_SYSTEM) { |
| 742 | /* header for write data commands */ |
| 743 | ndw += ncmds * 4; |
| 744 | |
| 745 | /* body of write data command */ |
| 746 | ndw += nptes * 2; |
| 747 | |
| 748 | } else { |
| 749 | /* set page commands needed */ |
| 750 | ndw += ncmds * 10; |
| 751 | |
| 752 | /* two extra commands for begin/end of fragment */ |
| 753 | ndw += 2 * 10; |
| 754 | } |
| 755 | |
| 756 | /* update too big for an IB */ |
| 757 | if (ndw > 0xfffff) |
| 758 | return -ENOMEM; |
| 759 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 760 | ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); |
| 761 | if (!ib) |
| 762 | return -ENOMEM; |
| 763 | |
| 764 | r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); |
| 765 | if (r) { |
| 766 | kfree(ib); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 767 | return r; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 768 | } |
| 769 | |
| 770 | ib->length_dw = 0; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 771 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 772 | r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 773 | mapping->it.last + 1, addr + mapping->offset, |
| 774 | flags, gtt_flags); |
| 775 | |
| 776 | if (r) { |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 777 | amdgpu_ib_free(adev, ib); |
| 778 | kfree(ib); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 779 | return r; |
| 780 | } |
| 781 | |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 782 | amdgpu_vm_pad_ib(adev, ib); |
| 783 | WARN_ON(ib->length_dw > ndw); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 784 | r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, |
| 785 | &amdgpu_vm_free_job, |
| 786 | AMDGPU_FENCE_OWNER_VM, |
| 787 | &f); |
| 788 | if (r) |
| 789 | goto error_free; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 790 | |
Christian König | bf60efd | 2015-09-04 10:47:56 +0200 | [diff] [blame] | 791 | amdgpu_bo_fence(vm->page_directory, f, true); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 792 | if (fence) { |
| 793 | fence_put(*fence); |
| 794 | *fence = fence_get(f); |
| 795 | } |
Chunming Zhou | 281b422 | 2015-08-12 12:58:31 +0800 | [diff] [blame] | 796 | fence_put(f); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 797 | if (!amdgpu_enable_scheduler) { |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 798 | amdgpu_ib_free(adev, ib); |
| 799 | kfree(ib); |
| 800 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 801 | return 0; |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 802 | |
| 803 | error_free: |
Chunming Zhou | d5fc5e8 | 2015-07-21 16:52:10 +0800 | [diff] [blame] | 804 | amdgpu_ib_free(adev, ib); |
| 805 | kfree(ib); |
Chunming Zhou | 4af9f07 | 2015-08-03 12:57:31 +0800 | [diff] [blame] | 806 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 807 | } |
| 808 | |
| 809 | /** |
| 810 | * amdgpu_vm_bo_update - update all BO mappings in the vm page table |
| 811 | * |
| 812 | * @adev: amdgpu_device pointer |
| 813 | * @bo_va: requested BO and VM object |
| 814 | * @mem: ttm mem |
| 815 | * |
| 816 | * Fill in the page table entries for @bo_va. |
| 817 | * Returns 0 for success, -EINVAL for failure. |
| 818 | * |
| 819 | * Object have to be reserved and mutex must be locked! |
| 820 | */ |
| 821 | int amdgpu_vm_bo_update(struct amdgpu_device *adev, |
| 822 | struct amdgpu_bo_va *bo_va, |
| 823 | struct ttm_mem_reg *mem) |
| 824 | { |
| 825 | struct amdgpu_vm *vm = bo_va->vm; |
| 826 | struct amdgpu_bo_va_mapping *mapping; |
| 827 | uint32_t flags; |
| 828 | uint64_t addr; |
| 829 | int r; |
| 830 | |
| 831 | if (mem) { |
Christian König | b7d698d | 2015-09-07 12:32:09 +0200 | [diff] [blame] | 832 | addr = (u64)mem->start << PAGE_SHIFT; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 833 | if (mem->mem_type != TTM_PL_TT) |
| 834 | addr += adev->vm_manager.vram_base_offset; |
| 835 | } else { |
| 836 | addr = 0; |
| 837 | } |
| 838 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 839 | flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); |
| 840 | |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 841 | spin_lock(&vm->status_lock); |
| 842 | if (!list_empty(&bo_va->vm_status)) |
| 843 | list_splice_init(&bo_va->valids, &bo_va->invalids); |
| 844 | spin_unlock(&vm->status_lock); |
| 845 | |
| 846 | list_for_each_entry(mapping, &bo_va->invalids, list) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 847 | r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, |
| 848 | flags, &bo_va->last_pt_update); |
| 849 | if (r) |
| 850 | return r; |
| 851 | } |
| 852 | |
Christian König | d6c10f6 | 2015-09-28 12:00:23 +0200 | [diff] [blame] | 853 | if (trace_amdgpu_vm_bo_mapping_enabled()) { |
| 854 | list_for_each_entry(mapping, &bo_va->valids, list) |
| 855 | trace_amdgpu_vm_bo_mapping(mapping); |
| 856 | |
| 857 | list_for_each_entry(mapping, &bo_va->invalids, list) |
| 858 | trace_amdgpu_vm_bo_mapping(mapping); |
| 859 | } |
| 860 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 861 | spin_lock(&vm->status_lock); |
monk.liu | 6d1d0ef | 2015-08-14 13:36:41 +0800 | [diff] [blame] | 862 | list_splice_init(&bo_va->invalids, &bo_va->valids); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 863 | list_del_init(&bo_va->vm_status); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 864 | if (!mem) |
| 865 | list_add(&bo_va->vm_status, &vm->cleared); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 866 | spin_unlock(&vm->status_lock); |
| 867 | |
| 868 | return 0; |
| 869 | } |
| 870 | |
| 871 | /** |
| 872 | * amdgpu_vm_clear_freed - clear freed BOs in the PT |
| 873 | * |
| 874 | * @adev: amdgpu_device pointer |
| 875 | * @vm: requested vm |
| 876 | * |
| 877 | * Make sure all freed BOs are cleared in the PT. |
| 878 | * Returns 0 for success. |
| 879 | * |
| 880 | * PTs have to be reserved and mutex must be locked! |
| 881 | */ |
| 882 | int amdgpu_vm_clear_freed(struct amdgpu_device *adev, |
| 883 | struct amdgpu_vm *vm) |
| 884 | { |
| 885 | struct amdgpu_bo_va_mapping *mapping; |
| 886 | int r; |
| 887 | |
| 888 | while (!list_empty(&vm->freed)) { |
| 889 | mapping = list_first_entry(&vm->freed, |
| 890 | struct amdgpu_bo_va_mapping, list); |
| 891 | list_del(&mapping->list); |
| 892 | |
| 893 | r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL); |
| 894 | kfree(mapping); |
| 895 | if (r) |
| 896 | return r; |
| 897 | |
| 898 | } |
| 899 | return 0; |
| 900 | |
| 901 | } |
| 902 | |
| 903 | /** |
| 904 | * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT |
| 905 | * |
| 906 | * @adev: amdgpu_device pointer |
| 907 | * @vm: requested vm |
| 908 | * |
| 909 | * Make sure all invalidated BOs are cleared in the PT. |
| 910 | * Returns 0 for success. |
| 911 | * |
| 912 | * PTs have to be reserved and mutex must be locked! |
| 913 | */ |
| 914 | int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, |
monk.liu | cfe2c97 | 2015-05-26 15:01:54 +0800 | [diff] [blame] | 915 | struct amdgpu_vm *vm, struct amdgpu_sync *sync) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 916 | { |
monk.liu | cfe2c97 | 2015-05-26 15:01:54 +0800 | [diff] [blame] | 917 | struct amdgpu_bo_va *bo_va = NULL; |
Christian König | 91e1a52 | 2015-07-06 22:06:40 +0200 | [diff] [blame] | 918 | int r = 0; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 919 | |
| 920 | spin_lock(&vm->status_lock); |
| 921 | while (!list_empty(&vm->invalidated)) { |
| 922 | bo_va = list_first_entry(&vm->invalidated, |
| 923 | struct amdgpu_bo_va, vm_status); |
| 924 | spin_unlock(&vm->status_lock); |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 925 | mutex_lock(&bo_va->mutex); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 926 | r = amdgpu_vm_bo_update(adev, bo_va, NULL); |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 927 | mutex_unlock(&bo_va->mutex); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 928 | if (r) |
| 929 | return r; |
| 930 | |
| 931 | spin_lock(&vm->status_lock); |
| 932 | } |
| 933 | spin_unlock(&vm->status_lock); |
| 934 | |
monk.liu | cfe2c97 | 2015-05-26 15:01:54 +0800 | [diff] [blame] | 935 | if (bo_va) |
Chunming Zhou | bb1e38a4 | 2015-08-03 18:19:38 +0800 | [diff] [blame] | 936 | r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); |
Christian König | 91e1a52 | 2015-07-06 22:06:40 +0200 | [diff] [blame] | 937 | |
| 938 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 939 | } |
| 940 | |
| 941 | /** |
| 942 | * amdgpu_vm_bo_add - add a bo to a specific vm |
| 943 | * |
| 944 | * @adev: amdgpu_device pointer |
| 945 | * @vm: requested vm |
| 946 | * @bo: amdgpu buffer object |
| 947 | * |
| 948 | * Add @bo into the requested vm (cayman+). |
| 949 | * Add @bo to the list of bos associated with the vm |
| 950 | * Returns newly added bo_va or NULL for failure |
| 951 | * |
| 952 | * Object has to be reserved! |
| 953 | */ |
| 954 | struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, |
| 955 | struct amdgpu_vm *vm, |
| 956 | struct amdgpu_bo *bo) |
| 957 | { |
| 958 | struct amdgpu_bo_va *bo_va; |
| 959 | |
| 960 | bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); |
| 961 | if (bo_va == NULL) { |
| 962 | return NULL; |
| 963 | } |
| 964 | bo_va->vm = vm; |
| 965 | bo_va->bo = bo; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 966 | bo_va->ref_count = 1; |
| 967 | INIT_LIST_HEAD(&bo_va->bo_list); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 968 | INIT_LIST_HEAD(&bo_va->valids); |
| 969 | INIT_LIST_HEAD(&bo_va->invalids); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 970 | INIT_LIST_HEAD(&bo_va->vm_status); |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 971 | mutex_init(&bo_va->mutex); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 972 | list_add_tail(&bo_va->bo_list, &bo->va); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 973 | |
| 974 | return bo_va; |
| 975 | } |
| 976 | |
| 977 | /** |
| 978 | * amdgpu_vm_bo_map - map bo inside a vm |
| 979 | * |
| 980 | * @adev: amdgpu_device pointer |
| 981 | * @bo_va: bo_va to store the address |
| 982 | * @saddr: where to map the BO |
| 983 | * @offset: requested offset in the BO |
| 984 | * @flags: attributes of pages (read/write/valid/etc.) |
| 985 | * |
| 986 | * Add a mapping of the BO at the specefied addr into the VM. |
| 987 | * Returns 0 for success, error for failure. |
| 988 | * |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 989 | * Object has to be reserved and unreserved outside! |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 990 | */ |
| 991 | int amdgpu_vm_bo_map(struct amdgpu_device *adev, |
| 992 | struct amdgpu_bo_va *bo_va, |
| 993 | uint64_t saddr, uint64_t offset, |
| 994 | uint64_t size, uint32_t flags) |
| 995 | { |
| 996 | struct amdgpu_bo_va_mapping *mapping; |
| 997 | struct amdgpu_vm *vm = bo_va->vm; |
| 998 | struct interval_tree_node *it; |
| 999 | unsigned last_pfn, pt_idx; |
| 1000 | uint64_t eaddr; |
| 1001 | int r; |
| 1002 | |
Christian König | 0be52de | 2015-05-18 14:37:27 +0200 | [diff] [blame] | 1003 | /* validate the parameters */ |
| 1004 | if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 1005 | size == 0 || size & AMDGPU_GPU_PAGE_MASK) |
Christian König | 0be52de | 2015-05-18 14:37:27 +0200 | [diff] [blame] | 1006 | return -EINVAL; |
Christian König | 0be52de | 2015-05-18 14:37:27 +0200 | [diff] [blame] | 1007 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1008 | /* make sure object fit at this offset */ |
| 1009 | eaddr = saddr + size; |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 1010 | if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1011 | return -EINVAL; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1012 | |
| 1013 | last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; |
| 1014 | if (last_pfn > adev->vm_manager.max_pfn) { |
| 1015 | dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n", |
| 1016 | last_pfn, adev->vm_manager.max_pfn); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1017 | return -EINVAL; |
| 1018 | } |
| 1019 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1020 | saddr /= AMDGPU_GPU_PAGE_SIZE; |
| 1021 | eaddr /= AMDGPU_GPU_PAGE_SIZE; |
| 1022 | |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1023 | spin_lock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1024 | it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1025 | spin_unlock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1026 | if (it) { |
| 1027 | struct amdgpu_bo_va_mapping *tmp; |
| 1028 | tmp = container_of(it, struct amdgpu_bo_va_mapping, it); |
| 1029 | /* bo and tmp overlap, invalid addr */ |
| 1030 | dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " |
| 1031 | "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, |
| 1032 | tmp->it.start, tmp->it.last + 1); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1033 | r = -EINVAL; |
Chunming Zhou | f48b265 | 2015-10-16 14:06:19 +0800 | [diff] [blame] | 1034 | goto error; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1035 | } |
| 1036 | |
| 1037 | mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); |
| 1038 | if (!mapping) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1039 | r = -ENOMEM; |
Chunming Zhou | f48b265 | 2015-10-16 14:06:19 +0800 | [diff] [blame] | 1040 | goto error; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1041 | } |
| 1042 | |
| 1043 | INIT_LIST_HEAD(&mapping->list); |
| 1044 | mapping->it.start = saddr; |
| 1045 | mapping->it.last = eaddr - 1; |
| 1046 | mapping->offset = offset; |
| 1047 | mapping->flags = flags; |
| 1048 | |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1049 | mutex_lock(&bo_va->mutex); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1050 | list_add(&mapping->list, &bo_va->invalids); |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1051 | mutex_unlock(&bo_va->mutex); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1052 | spin_lock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1053 | interval_tree_insert(&mapping->it, &vm->va); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1054 | spin_unlock(&vm->it_lock); |
Christian König | 93e3e43 | 2015-06-09 16:58:33 +0200 | [diff] [blame] | 1055 | trace_amdgpu_vm_bo_map(bo_va, mapping); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1056 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1057 | /* Make sure the page tables are allocated */ |
| 1058 | saddr >>= amdgpu_vm_block_size; |
| 1059 | eaddr >>= amdgpu_vm_block_size; |
| 1060 | |
| 1061 | BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev)); |
| 1062 | |
| 1063 | if (eaddr > vm->max_pde_used) |
| 1064 | vm->max_pde_used = eaddr; |
| 1065 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1066 | /* walk over the address space and allocate the page tables */ |
| 1067 | for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) { |
Christian König | bf60efd | 2015-09-04 10:47:56 +0200 | [diff] [blame] | 1068 | struct reservation_object *resv = vm->page_directory->tbo.resv; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1069 | struct amdgpu_bo *pt; |
| 1070 | |
| 1071 | if (vm->page_tables[pt_idx].bo) |
| 1072 | continue; |
| 1073 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1074 | r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, |
| 1075 | AMDGPU_GPU_PAGE_SIZE, true, |
Alex Deucher | 857d913 | 2015-08-27 00:14:16 -0400 | [diff] [blame] | 1076 | AMDGPU_GEM_DOMAIN_VRAM, |
| 1077 | AMDGPU_GEM_CREATE_NO_CPU_ACCESS, |
Christian König | bf60efd | 2015-09-04 10:47:56 +0200 | [diff] [blame] | 1078 | NULL, resv, &pt); |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 1079 | if (r) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1080 | goto error_free; |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 1081 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1082 | r = amdgpu_vm_clear_bo(adev, pt); |
| 1083 | if (r) { |
| 1084 | amdgpu_bo_unref(&pt); |
| 1085 | goto error_free; |
| 1086 | } |
| 1087 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1088 | vm->page_tables[pt_idx].addr = 0; |
| 1089 | vm->page_tables[pt_idx].bo = pt; |
| 1090 | } |
| 1091 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1092 | return 0; |
| 1093 | |
| 1094 | error_free: |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1095 | list_del(&mapping->list); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1096 | spin_lock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1097 | interval_tree_remove(&mapping->it, &vm->va); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1098 | spin_unlock(&vm->it_lock); |
Christian König | 93e3e43 | 2015-06-09 16:58:33 +0200 | [diff] [blame] | 1099 | trace_amdgpu_vm_bo_unmap(bo_va, mapping); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1100 | kfree(mapping); |
| 1101 | |
Chunming Zhou | f48b265 | 2015-10-16 14:06:19 +0800 | [diff] [blame] | 1102 | error: |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1103 | return r; |
| 1104 | } |
| 1105 | |
| 1106 | /** |
| 1107 | * amdgpu_vm_bo_unmap - remove bo mapping from vm |
| 1108 | * |
| 1109 | * @adev: amdgpu_device pointer |
| 1110 | * @bo_va: bo_va to remove the address from |
| 1111 | * @saddr: where to the BO is mapped |
| 1112 | * |
| 1113 | * Remove a mapping of the BO at the specefied addr from the VM. |
| 1114 | * Returns 0 for success, error for failure. |
| 1115 | * |
Chunming Zhou | 49b02b1 | 2015-11-13 14:18:38 +0800 | [diff] [blame] | 1116 | * Object has to be reserved and unreserved outside! |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1117 | */ |
| 1118 | int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, |
| 1119 | struct amdgpu_bo_va *bo_va, |
| 1120 | uint64_t saddr) |
| 1121 | { |
| 1122 | struct amdgpu_bo_va_mapping *mapping; |
| 1123 | struct amdgpu_vm *vm = bo_va->vm; |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1124 | bool valid = true; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1125 | |
Christian König | 6c7fc50 | 2015-06-05 20:56:17 +0200 | [diff] [blame] | 1126 | saddr /= AMDGPU_GPU_PAGE_SIZE; |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1127 | mutex_lock(&bo_va->mutex); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1128 | list_for_each_entry(mapping, &bo_va->valids, list) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1129 | if (mapping->it.start == saddr) |
| 1130 | break; |
| 1131 | } |
| 1132 | |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1133 | if (&mapping->list == &bo_va->valids) { |
| 1134 | valid = false; |
| 1135 | |
| 1136 | list_for_each_entry(mapping, &bo_va->invalids, list) { |
| 1137 | if (mapping->it.start == saddr) |
| 1138 | break; |
| 1139 | } |
| 1140 | |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1141 | if (&mapping->list == &bo_va->invalids) { |
| 1142 | mutex_unlock(&bo_va->mutex); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1143 | return -ENOENT; |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1144 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1145 | } |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1146 | mutex_unlock(&bo_va->mutex); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1147 | list_del(&mapping->list); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1148 | spin_lock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1149 | interval_tree_remove(&mapping->it, &vm->va); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1150 | spin_unlock(&vm->it_lock); |
Christian König | 93e3e43 | 2015-06-09 16:58:33 +0200 | [diff] [blame] | 1151 | trace_amdgpu_vm_bo_unmap(bo_va, mapping); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1152 | |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1153 | if (valid) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1154 | list_add(&mapping->list, &vm->freed); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1155 | else |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1156 | kfree(mapping); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1157 | |
| 1158 | return 0; |
| 1159 | } |
| 1160 | |
| 1161 | /** |
| 1162 | * amdgpu_vm_bo_rmv - remove a bo to a specific vm |
| 1163 | * |
| 1164 | * @adev: amdgpu_device pointer |
| 1165 | * @bo_va: requested bo_va |
| 1166 | * |
| 1167 | * Remove @bo_va->bo from the requested vm (cayman+). |
| 1168 | * |
| 1169 | * Object have to be reserved! |
| 1170 | */ |
| 1171 | void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, |
| 1172 | struct amdgpu_bo_va *bo_va) |
| 1173 | { |
| 1174 | struct amdgpu_bo_va_mapping *mapping, *next; |
| 1175 | struct amdgpu_vm *vm = bo_va->vm; |
| 1176 | |
| 1177 | list_del(&bo_va->bo_list); |
| 1178 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1179 | spin_lock(&vm->status_lock); |
| 1180 | list_del(&bo_va->vm_status); |
| 1181 | spin_unlock(&vm->status_lock); |
| 1182 | |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1183 | list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1184 | list_del(&mapping->list); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1185 | spin_lock(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1186 | interval_tree_remove(&mapping->it, &vm->va); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1187 | spin_unlock(&vm->it_lock); |
Christian König | 93e3e43 | 2015-06-09 16:58:33 +0200 | [diff] [blame] | 1188 | trace_amdgpu_vm_bo_unmap(bo_va, mapping); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1189 | list_add(&mapping->list, &vm->freed); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1190 | } |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1191 | list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { |
| 1192 | list_del(&mapping->list); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1193 | spin_lock(&vm->it_lock); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1194 | interval_tree_remove(&mapping->it, &vm->va); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1195 | spin_unlock(&vm->it_lock); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1196 | kfree(mapping); |
| 1197 | } |
Chunming Zhou | bb1e38a4 | 2015-08-03 18:19:38 +0800 | [diff] [blame] | 1198 | fence_put(bo_va->last_pt_update); |
Chunming Zhou | 69b576a | 2015-11-18 11:17:39 +0800 | [diff] [blame] | 1199 | mutex_destroy(&bo_va->mutex); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1200 | kfree(bo_va); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1201 | } |
| 1202 | |
| 1203 | /** |
| 1204 | * amdgpu_vm_bo_invalidate - mark the bo as invalid |
| 1205 | * |
| 1206 | * @adev: amdgpu_device pointer |
| 1207 | * @vm: requested vm |
| 1208 | * @bo: amdgpu buffer object |
| 1209 | * |
| 1210 | * Mark @bo as invalid (cayman+). |
| 1211 | */ |
| 1212 | void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, |
| 1213 | struct amdgpu_bo *bo) |
| 1214 | { |
| 1215 | struct amdgpu_bo_va *bo_va; |
| 1216 | |
| 1217 | list_for_each_entry(bo_va, &bo->va, bo_list) { |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1218 | spin_lock(&bo_va->vm->status_lock); |
| 1219 | if (list_empty(&bo_va->vm_status)) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1220 | list_add(&bo_va->vm_status, &bo_va->vm->invalidated); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1221 | spin_unlock(&bo_va->vm->status_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1222 | } |
| 1223 | } |
| 1224 | |
| 1225 | /** |
| 1226 | * amdgpu_vm_init - initialize a vm instance |
| 1227 | * |
| 1228 | * @adev: amdgpu_device pointer |
| 1229 | * @vm: requested vm |
| 1230 | * |
| 1231 | * Init @vm fields (cayman+). |
| 1232 | */ |
| 1233 | int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) |
| 1234 | { |
| 1235 | const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, |
| 1236 | AMDGPU_VM_PTE_COUNT * 8); |
| 1237 | unsigned pd_size, pd_entries, pts_size; |
| 1238 | int i, r; |
| 1239 | |
| 1240 | for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { |
| 1241 | vm->ids[i].id = 0; |
| 1242 | vm->ids[i].flushed_updates = NULL; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1243 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1244 | vm->va = RB_ROOT; |
| 1245 | spin_lock_init(&vm->status_lock); |
| 1246 | INIT_LIST_HEAD(&vm->invalidated); |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 1247 | INIT_LIST_HEAD(&vm->cleared); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1248 | INIT_LIST_HEAD(&vm->freed); |
Chunming Zhou | c25867d | 2015-11-13 13:32:01 +0800 | [diff] [blame] | 1249 | spin_lock_init(&vm->it_lock); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1250 | pd_size = amdgpu_vm_directory_size(adev); |
| 1251 | pd_entries = amdgpu_vm_num_pdes(adev); |
| 1252 | |
| 1253 | /* allocate page table array */ |
| 1254 | pts_size = pd_entries * sizeof(struct amdgpu_vm_pt); |
| 1255 | vm->page_tables = kzalloc(pts_size, GFP_KERNEL); |
| 1256 | if (vm->page_tables == NULL) { |
| 1257 | DRM_ERROR("Cannot allocate memory for page table array\n"); |
| 1258 | return -ENOMEM; |
| 1259 | } |
| 1260 | |
Bas Nieuwenhuizen | 05906de | 2015-08-14 20:08:40 +0200 | [diff] [blame] | 1261 | vm->page_directory_fence = NULL; |
| 1262 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1263 | r = amdgpu_bo_create(adev, pd_size, align, true, |
Alex Deucher | 857d913 | 2015-08-27 00:14:16 -0400 | [diff] [blame] | 1264 | AMDGPU_GEM_DOMAIN_VRAM, |
| 1265 | AMDGPU_GEM_CREATE_NO_CPU_ACCESS, |
Christian König | 72d7668 | 2015-09-03 17:34:59 +0200 | [diff] [blame] | 1266 | NULL, NULL, &vm->page_directory); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1267 | if (r) |
| 1268 | return r; |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 1269 | r = amdgpu_bo_reserve(vm->page_directory, false); |
| 1270 | if (r) { |
| 1271 | amdgpu_bo_unref(&vm->page_directory); |
| 1272 | vm->page_directory = NULL; |
| 1273 | return r; |
| 1274 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1275 | r = amdgpu_vm_clear_bo(adev, vm->page_directory); |
Chunming Zhou | ef9f0a8 | 2015-11-13 13:43:22 +0800 | [diff] [blame] | 1276 | amdgpu_bo_unreserve(vm->page_directory); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1277 | if (r) { |
| 1278 | amdgpu_bo_unref(&vm->page_directory); |
| 1279 | vm->page_directory = NULL; |
| 1280 | return r; |
| 1281 | } |
| 1282 | |
| 1283 | return 0; |
| 1284 | } |
| 1285 | |
| 1286 | /** |
| 1287 | * amdgpu_vm_fini - tear down a vm instance |
| 1288 | * |
| 1289 | * @adev: amdgpu_device pointer |
| 1290 | * @vm: requested vm |
| 1291 | * |
| 1292 | * Tear down @vm (cayman+). |
| 1293 | * Unbind the VM and remove all bos from the vm bo list |
| 1294 | */ |
| 1295 | void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) |
| 1296 | { |
| 1297 | struct amdgpu_bo_va_mapping *mapping, *tmp; |
| 1298 | int i; |
| 1299 | |
| 1300 | if (!RB_EMPTY_ROOT(&vm->va)) { |
| 1301 | dev_err(adev->dev, "still active bo inside vm\n"); |
| 1302 | } |
| 1303 | rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { |
| 1304 | list_del(&mapping->list); |
| 1305 | interval_tree_remove(&mapping->it, &vm->va); |
| 1306 | kfree(mapping); |
| 1307 | } |
| 1308 | list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { |
| 1309 | list_del(&mapping->list); |
| 1310 | kfree(mapping); |
| 1311 | } |
| 1312 | |
| 1313 | for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) |
| 1314 | amdgpu_bo_unref(&vm->page_tables[i].bo); |
| 1315 | kfree(vm->page_tables); |
| 1316 | |
| 1317 | amdgpu_bo_unref(&vm->page_directory); |
Bas Nieuwenhuizen | 05906de | 2015-08-14 20:08:40 +0200 | [diff] [blame] | 1318 | fence_put(vm->page_directory_fence); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1319 | for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 1320 | unsigned id = vm->ids[i].id; |
| 1321 | |
| 1322 | atomic_long_cmpxchg(&adev->vm_manager.ids[id].owner, |
| 1323 | (long)vm, 0); |
Chunming Zhou | 3c62338 | 2015-08-20 18:33:59 +0800 | [diff] [blame] | 1324 | fence_put(vm->ids[i].flushed_updates); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1325 | } |
| 1326 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1327 | } |
Christian König | ea89f8c | 2015-11-15 20:52:06 +0100 | [diff] [blame] | 1328 | |
| 1329 | /** |
| 1330 | * amdgpu_vm_manager_fini - cleanup VM manager |
| 1331 | * |
| 1332 | * @adev: amdgpu_device pointer |
| 1333 | * |
| 1334 | * Cleanup the VM manager and free resources. |
| 1335 | */ |
| 1336 | void amdgpu_vm_manager_fini(struct amdgpu_device *adev) |
| 1337 | { |
| 1338 | unsigned i; |
| 1339 | |
| 1340 | for (i = 0; i < AMDGPU_NUM_VM; ++i) |
Christian König | 1c16c0a | 2015-11-14 21:31:40 +0100 | [diff] [blame] | 1341 | fence_put(adev->vm_manager.ids[i].active); |
Christian König | ea89f8c | 2015-11-15 20:52:06 +0100 | [diff] [blame] | 1342 | } |