blob: 17ae6afdef7010b2e638e4916ed2df6c56cd1fe3 [file] [log] [blame]
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
Chris Wilsonf54d1862016-10-25 13:00:45 +010028#include <linux/dma-fence-array.h>
Christian Königa9f87f62017-03-30 14:03:59 +020029#include <linux/interval_tree_generic.h>
Felix Kuehling02208442017-08-25 20:40:26 -040030#include <linux/idr.h>
Alex Deucherd38ceaf2015-04-20 16:55:21 -040031#include <drm/drmP.h>
32#include <drm/amdgpu_drm.h>
33#include "amdgpu.h"
34#include "amdgpu_trace.h"
35
36/*
Felix Kuehling02208442017-08-25 20:40:26 -040037 * PASID manager
38 *
39 * PASIDs are global address space identifiers that can be shared
40 * between the GPU, an IOMMU and the driver. VMs on different devices
41 * may use the same PASID if they share the same address
42 * space. Therefore PASIDs are allocated using a global IDA. VMs are
43 * looked up from the PASID per amdgpu_device.
44 */
45static DEFINE_IDA(amdgpu_vm_pasid_ida);
46
47/**
48 * amdgpu_vm_alloc_pasid - Allocate a PASID
49 * @bits: Maximum width of the PASID in bits, must be at least 1
50 *
51 * Allocates a PASID of the given width while keeping smaller PASIDs
52 * available if possible.
53 *
54 * Returns a positive integer on success. Returns %-EINVAL if bits==0.
55 * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
56 * memory allocation failure.
57 */
58int amdgpu_vm_alloc_pasid(unsigned int bits)
59{
60 int pasid = -EINVAL;
61
62 for (bits = min(bits, 31U); bits > 0; bits--) {
63 pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
64 1U << (bits - 1), 1U << bits,
65 GFP_KERNEL);
66 if (pasid != -ENOSPC)
67 break;
68 }
69
70 return pasid;
71}
72
73/**
74 * amdgpu_vm_free_pasid - Free a PASID
75 * @pasid: PASID to free
76 */
77void amdgpu_vm_free_pasid(unsigned int pasid)
78{
79 ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
80}
81
82/*
Alex Deucherd38ceaf2015-04-20 16:55:21 -040083 * GPUVM
84 * GPUVM is similar to the legacy gart on older asics, however
85 * rather than there being a single global gart table
86 * for the entire GPU, there are multiple VM page tables active
87 * at any given time. The VM page tables can contain a mix
88 * vram pages and system memory pages and system memory pages
89 * can be mapped as snooped (cached system pages) or unsnooped
90 * (uncached system pages).
91 * Each VM has an ID associated with it and there is a page table
92 * associated with each VMID. When execting a command buffer,
93 * the kernel tells the the ring what VMID to use for that command
94 * buffer. VMIDs are allocated dynamically as commands are submitted.
95 * The userspace drivers maintain their own address space and the kernel
96 * sets up their pages tables accordingly when they submit their
97 * command buffers and a VMID is assigned.
98 * Cayman/Trinity support up to 8 active VMs at any given time;
99 * SI supports 16.
100 */
101
Christian Königa9f87f62017-03-30 14:03:59 +0200102#define START(node) ((node)->start)
103#define LAST(node) ((node)->last)
104
105INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
106 START, LAST, static, amdgpu_vm_it)
107
108#undef START
109#undef LAST
110
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400111/* Local structure. Encapsulate some VM table update parameters to reduce
112 * the number of function parameters
113 */
Christian König29efc4f2016-08-04 14:52:50 +0200114struct amdgpu_pte_update_params {
Christian König27c5f362016-08-04 15:02:49 +0200115 /* amdgpu device we do this update for */
116 struct amdgpu_device *adev;
Christian König49ac8a22016-10-13 15:09:08 +0200117 /* optional amdgpu_vm we do this update for */
118 struct amdgpu_vm *vm;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400119 /* address where to copy page table entries from */
120 uint64_t src;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400121 /* indirect buffer to fill with commands */
122 struct amdgpu_ib *ib;
Christian Königafef8b82016-08-12 13:29:18 +0200123 /* Function which actually does the update */
124 void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
125 uint64_t addr, unsigned count, uint32_t incr,
Chunming Zhou6b777602016-09-21 16:19:19 +0800126 uint64_t flags);
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -0400127 /* The next two are used during VM update by CPU
128 * DMA addresses to use for mapping
129 * Kernel pointer of PD/PT BO that needs to be updated
130 */
131 dma_addr_t *pages_addr;
132 void *kptr;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400133};
134
Christian König284710f2017-01-30 11:09:31 +0100135/* Helper to disable partial resident texture feature from a fence callback */
136struct amdgpu_prt_cb {
137 struct amdgpu_device *adev;
138 struct dma_fence_cb cb;
139};
140
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400141/**
Christian König50783142017-11-27 14:01:51 +0100142 * amdgpu_vm_level_shift - return the addr shift for each level
143 *
144 * @adev: amdgpu_device pointer
145 *
146 * Returns the number of bits the pfn needs to be right shifted for a level.
147 */
148static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
149 unsigned level)
150{
151 if (level != adev->vm_manager.num_level)
152 return 9 * (adev->vm_manager.num_level - level - 1) +
153 adev->vm_manager.block_size;
154 else
155 /* For the page tables on the leaves */
156 return 0;
157}
158
159/**
Christian König72a7ec52016-10-19 11:03:57 +0200160 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400161 *
162 * @adev: amdgpu_device pointer
163 *
Christian König72a7ec52016-10-19 11:03:57 +0200164 * Calculate the number of entries in a page directory or page table.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400165 */
Christian König72a7ec52016-10-19 11:03:57 +0200166static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
167 unsigned level)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400168{
Christian König0410c5e2017-11-20 14:29:01 +0100169 unsigned shift = amdgpu_vm_level_shift(adev, 0);
170
Christian König72a7ec52016-10-19 11:03:57 +0200171 if (level == 0)
172 /* For the root directory */
Christian König0410c5e2017-11-20 14:29:01 +0100173 return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
174 else if (level != adev->vm_manager.num_level)
175 /* Everything in between */
176 return 512;
177 else
Christian König72a7ec52016-10-19 11:03:57 +0200178 /* For the page tables on the leaves */
Zhang, Jerry36b32a62017-03-29 16:08:32 +0800179 return AMDGPU_VM_PTE_COUNT(adev);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400180}
181
182/**
Christian König72a7ec52016-10-19 11:03:57 +0200183 * amdgpu_vm_bo_size - returns the size of the BOs in bytes
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400184 *
185 * @adev: amdgpu_device pointer
186 *
Christian König72a7ec52016-10-19 11:03:57 +0200187 * Calculate the size of the BO for a page directory or page table in bytes.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400188 */
Christian König72a7ec52016-10-19 11:03:57 +0200189static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400190{
Christian König72a7ec52016-10-19 11:03:57 +0200191 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400192}
193
194/**
Christian König56467eb2015-12-11 15:16:32 +0100195 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400196 *
197 * @vm: vm providing the BOs
Christian König3c0eea62015-12-11 14:39:05 +0100198 * @validated: head of validation list
Christian König56467eb2015-12-11 15:16:32 +0100199 * @entry: entry to add
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400200 *
201 * Add the page directory to the list of BOs to
Christian König56467eb2015-12-11 15:16:32 +0100202 * validate for command submission.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400203 */
Christian König56467eb2015-12-11 15:16:32 +0100204void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
205 struct list_head *validated,
206 struct amdgpu_bo_list_entry *entry)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400207{
Christian König3f3333f2017-08-03 14:02:13 +0200208 entry->robj = vm->root.base.bo;
Christian König56467eb2015-12-11 15:16:32 +0100209 entry->priority = 0;
Christian König67003a12016-10-12 14:46:26 +0200210 entry->tv.bo = &entry->robj->tbo;
Christian König56467eb2015-12-11 15:16:32 +0100211 entry->tv.shared = true;
Christian König2f568db2016-02-23 12:36:59 +0100212 entry->user_pages = NULL;
Christian König56467eb2015-12-11 15:16:32 +0100213 list_add(&entry->tv.head, validated);
214}
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400215
Christian König56467eb2015-12-11 15:16:32 +0100216/**
Christian Königf7da30d2016-09-28 12:03:04 +0200217 * amdgpu_vm_validate_pt_bos - validate the page table BOs
Christian König56467eb2015-12-11 15:16:32 +0100218 *
Christian König5a712a82016-06-21 16:28:15 +0200219 * @adev: amdgpu device pointer
Christian König56467eb2015-12-11 15:16:32 +0100220 * @vm: vm providing the BOs
Christian Königf7da30d2016-09-28 12:03:04 +0200221 * @validate: callback to do the validation
222 * @param: parameter for the validation callback
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400223 *
Christian Königf7da30d2016-09-28 12:03:04 +0200224 * Validate the page table BOs on command submission if neccessary.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400225 */
Christian Königf7da30d2016-09-28 12:03:04 +0200226int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
227 int (*validate)(void *p, struct amdgpu_bo *bo),
228 void *param)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400229{
Christian König3f3333f2017-08-03 14:02:13 +0200230 struct ttm_bo_global *glob = adev->mman.bdev.glob;
231 int r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400232
Christian König3f3333f2017-08-03 14:02:13 +0200233 spin_lock(&vm->status_lock);
234 while (!list_empty(&vm->evicted)) {
235 struct amdgpu_vm_bo_base *bo_base;
236 struct amdgpu_bo *bo;
Christian König5a712a82016-06-21 16:28:15 +0200237
Christian König3f3333f2017-08-03 14:02:13 +0200238 bo_base = list_first_entry(&vm->evicted,
239 struct amdgpu_vm_bo_base,
240 vm_status);
241 spin_unlock(&vm->status_lock);
Christian Königeceb8a12016-01-11 15:35:21 +0100242
Christian König3f3333f2017-08-03 14:02:13 +0200243 bo = bo_base->bo;
244 BUG_ON(!bo);
245 if (bo->parent) {
246 r = validate(param, bo);
247 if (r)
248 return r;
Christian König34d7be52017-08-24 12:32:55 +0200249
Christian König3f3333f2017-08-03 14:02:13 +0200250 spin_lock(&glob->lru_lock);
251 ttm_bo_move_to_lru_tail(&bo->tbo);
252 if (bo->shadow)
253 ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
254 spin_unlock(&glob->lru_lock);
255 }
256
Christian König73fb16e2017-08-16 11:13:48 +0200257 if (bo->tbo.type == ttm_bo_type_kernel &&
258 vm->use_cpu_for_update) {
Christian König3f3333f2017-08-03 14:02:13 +0200259 r = amdgpu_bo_kmap(bo, NULL);
260 if (r)
261 return r;
262 }
263
264 spin_lock(&vm->status_lock);
Christian König73fb16e2017-08-16 11:13:48 +0200265 if (bo->tbo.type != ttm_bo_type_kernel)
266 list_move(&bo_base->vm_status, &vm->moved);
267 else
268 list_move(&bo_base->vm_status, &vm->relocated);
Christian König3f3333f2017-08-03 14:02:13 +0200269 }
270 spin_unlock(&vm->status_lock);
Christian König34d7be52017-08-24 12:32:55 +0200271
272 return 0;
273}
274
275/**
276 * amdgpu_vm_ready - check VM is ready for updates
277 *
Christian König34d7be52017-08-24 12:32:55 +0200278 * @vm: VM to check
279 *
280 * Check if all VM PDs/PTs are ready for updates
281 */
Christian König3f3333f2017-08-03 14:02:13 +0200282bool amdgpu_vm_ready(struct amdgpu_vm *vm)
Christian König34d7be52017-08-24 12:32:55 +0200283{
Christian König3f3333f2017-08-03 14:02:13 +0200284 bool ready;
Christian König34d7be52017-08-24 12:32:55 +0200285
Christian König3f3333f2017-08-03 14:02:13 +0200286 spin_lock(&vm->status_lock);
287 ready = list_empty(&vm->evicted);
288 spin_unlock(&vm->status_lock);
289
290 return ready;
Christian Königeceb8a12016-01-11 15:35:21 +0100291}
292
293/**
Christian Königf566ceb2016-10-27 20:04:38 +0200294 * amdgpu_vm_alloc_levels - allocate the PD/PT levels
295 *
296 * @adev: amdgpu_device pointer
297 * @vm: requested vm
298 * @saddr: start of the address range
299 * @eaddr: end of the address range
300 *
301 * Make sure the page directories and page tables are allocated
302 */
303static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
304 struct amdgpu_vm *vm,
305 struct amdgpu_vm_pt *parent,
306 uint64_t saddr, uint64_t eaddr,
307 unsigned level)
308{
Christian König50783142017-11-27 14:01:51 +0100309 unsigned shift = amdgpu_vm_level_shift(adev, level);
Christian Königf566ceb2016-10-27 20:04:38 +0200310 unsigned pt_idx, from, to;
311 int r;
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -0400312 u64 flags;
Yong Zhao51ac7ee2017-07-27 12:48:22 -0400313 uint64_t init_value = 0;
Christian Königf566ceb2016-10-27 20:04:38 +0200314
315 if (!parent->entries) {
316 unsigned num_entries = amdgpu_vm_num_entries(adev, level);
317
Michal Hocko20981052017-05-17 14:23:12 +0200318 parent->entries = kvmalloc_array(num_entries,
319 sizeof(struct amdgpu_vm_pt),
320 GFP_KERNEL | __GFP_ZERO);
Christian Königf566ceb2016-10-27 20:04:38 +0200321 if (!parent->entries)
322 return -ENOMEM;
323 memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
324 }
325
Felix Kuehling1866bac2017-03-28 20:36:12 -0400326 from = saddr >> shift;
327 to = eaddr >> shift;
328 if (from >= amdgpu_vm_num_entries(adev, level) ||
329 to >= amdgpu_vm_num_entries(adev, level))
330 return -EINVAL;
Christian Königf566ceb2016-10-27 20:04:38 +0200331
332 if (to > parent->last_entry_used)
333 parent->last_entry_used = to;
334
335 ++level;
Felix Kuehling1866bac2017-03-28 20:36:12 -0400336 saddr = saddr & ((1 << shift) - 1);
337 eaddr = eaddr & ((1 << shift) - 1);
Christian Königf566ceb2016-10-27 20:04:38 +0200338
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -0400339 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
340 AMDGPU_GEM_CREATE_VRAM_CLEARED;
341 if (vm->use_cpu_for_update)
342 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
343 else
344 flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
345 AMDGPU_GEM_CREATE_SHADOW);
346
Yong Zhao51ac7ee2017-07-27 12:48:22 -0400347 if (vm->pte_support_ats) {
Yong Zhao6d16dac2017-08-31 15:55:00 -0400348 init_value = AMDGPU_PTE_DEFAULT_ATC;
Yong Zhao51ac7ee2017-07-27 12:48:22 -0400349 if (level != adev->vm_manager.num_level - 1)
350 init_value |= AMDGPU_PDE_PTE;
Yong Zhao6d16dac2017-08-31 15:55:00 -0400351
Yong Zhao51ac7ee2017-07-27 12:48:22 -0400352 }
353
Christian Königf566ceb2016-10-27 20:04:38 +0200354 /* walk over the address space and allocate the page tables */
355 for (pt_idx = from; pt_idx <= to; ++pt_idx) {
Christian König3f3333f2017-08-03 14:02:13 +0200356 struct reservation_object *resv = vm->root.base.bo->tbo.resv;
Christian Königf566ceb2016-10-27 20:04:38 +0200357 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
358 struct amdgpu_bo *pt;
359
Christian König3f3333f2017-08-03 14:02:13 +0200360 if (!entry->base.bo) {
Christian Königf566ceb2016-10-27 20:04:38 +0200361 r = amdgpu_bo_create(adev,
362 amdgpu_vm_bo_size(adev, level),
363 AMDGPU_GPU_PAGE_SIZE, true,
364 AMDGPU_GEM_DOMAIN_VRAM,
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -0400365 flags,
Yong Zhao51ac7ee2017-07-27 12:48:22 -0400366 NULL, resv, init_value, &pt);
Christian Königf566ceb2016-10-27 20:04:38 +0200367 if (r)
368 return r;
369
Christian König0a096fb2017-07-12 10:01:48 +0200370 if (vm->use_cpu_for_update) {
371 r = amdgpu_bo_kmap(pt, NULL);
372 if (r) {
373 amdgpu_bo_unref(&pt);
374 return r;
375 }
376 }
377
Christian Königf566ceb2016-10-27 20:04:38 +0200378 /* Keep a reference to the root directory to avoid
379 * freeing them up in the wrong order.
380 */
Christian König0f2fc432017-08-31 10:46:20 +0200381 pt->parent = amdgpu_bo_ref(parent->base.bo);
Christian Königf566ceb2016-10-27 20:04:38 +0200382
Christian König3f3333f2017-08-03 14:02:13 +0200383 entry->base.vm = vm;
384 entry->base.bo = pt;
385 list_add_tail(&entry->base.bo_list, &pt->va);
Christian Königea097292017-08-09 14:15:46 +0200386 spin_lock(&vm->status_lock);
387 list_add(&entry->base.vm_status, &vm->relocated);
388 spin_unlock(&vm->status_lock);
Christian Königf566ceb2016-10-27 20:04:38 +0200389 entry->addr = 0;
390 }
391
392 if (level < adev->vm_manager.num_level) {
Felix Kuehling1866bac2017-03-28 20:36:12 -0400393 uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
394 uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
395 ((1 << shift) - 1);
396 r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
397 sub_eaddr, level);
Christian Königf566ceb2016-10-27 20:04:38 +0200398 if (r)
399 return r;
400 }
401 }
402
403 return 0;
404}
405
Christian König663e4572017-03-13 10:13:37 +0100406/**
407 * amdgpu_vm_alloc_pts - Allocate page tables.
408 *
409 * @adev: amdgpu_device pointer
410 * @vm: VM to allocate page tables for
411 * @saddr: Start address which needs to be allocated
412 * @size: Size from start address we need.
413 *
414 * Make sure the page tables are allocated.
415 */
416int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
417 struct amdgpu_vm *vm,
418 uint64_t saddr, uint64_t size)
419{
Felix Kuehling22770e52017-03-28 20:24:53 -0400420 uint64_t last_pfn;
Christian König663e4572017-03-13 10:13:37 +0100421 uint64_t eaddr;
Christian König663e4572017-03-13 10:13:37 +0100422
423 /* validate the parameters */
424 if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
425 return -EINVAL;
426
427 eaddr = saddr + size - 1;
428 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
429 if (last_pfn >= adev->vm_manager.max_pfn) {
Felix Kuehling22770e52017-03-28 20:24:53 -0400430 dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
Christian König663e4572017-03-13 10:13:37 +0100431 last_pfn, adev->vm_manager.max_pfn);
432 return -EINVAL;
433 }
434
435 saddr /= AMDGPU_GPU_PAGE_SIZE;
436 eaddr /= AMDGPU_GPU_PAGE_SIZE;
437
Christian Königf566ceb2016-10-27 20:04:38 +0200438 return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
Christian König663e4572017-03-13 10:13:37 +0100439}
440
Christian König641e9402017-04-03 13:59:25 +0200441/**
442 * amdgpu_vm_had_gpu_reset - check if reset occured since last use
443 *
444 * @adev: amdgpu_device pointer
445 * @id: VMID structure
446 *
447 * Check if GPU reset occured since last use of the VMID.
448 */
449static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
450 struct amdgpu_vm_id *id)
Chunming Zhou192b7dc2016-06-29 14:01:15 +0800451{
452 return id->current_gpu_reset_count !=
Christian König641e9402017-04-03 13:59:25 +0200453 atomic_read(&adev->gpu_reset_counter);
Chunming Zhou192b7dc2016-06-29 14:01:15 +0800454}
455
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800456static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
457{
458 return !!vm->reserved_vmid[vmhub];
459}
460
461/* idr_mgr->lock must be held */
462static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
463 struct amdgpu_ring *ring,
464 struct amdgpu_sync *sync,
465 struct dma_fence *fence,
466 struct amdgpu_job *job)
467{
468 struct amdgpu_device *adev = ring->adev;
469 unsigned vmhub = ring->funcs->vmhub;
470 uint64_t fence_context = adev->fence_context + ring->idx;
471 struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
472 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
473 struct dma_fence *updates = sync->last_vm_update;
474 int r = 0;
475 struct dma_fence *flushed, *tmp;
Christian König6f1ceab2017-07-11 16:59:21 +0200476 bool needs_flush = vm->use_cpu_for_update;
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800477
478 flushed = id->flushed_updates;
479 if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
480 (atomic64_read(&id->owner) != vm->client_id) ||
481 (job->vm_pd_addr != id->pd_gpu_addr) ||
482 (updates && (!flushed || updates->context != flushed->context ||
483 dma_fence_is_later(updates, flushed))) ||
484 (!id->last_flush || (id->last_flush->context != fence_context &&
485 !dma_fence_is_signaled(id->last_flush)))) {
486 needs_flush = true;
487 /* to prevent one context starved by another context */
488 id->pd_gpu_addr = 0;
489 tmp = amdgpu_sync_peek_fence(&id->active, ring);
490 if (tmp) {
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -0500491 r = amdgpu_sync_fence(adev, sync, tmp, false);
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800492 return r;
493 }
494 }
495
496 /* Good we can use this VMID. Remember this submission as
497 * user of the VMID.
498 */
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -0500499 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800500 if (r)
501 goto out;
502
503 if (updates && (!flushed || updates->context != flushed->context ||
504 dma_fence_is_later(updates, flushed))) {
505 dma_fence_put(id->flushed_updates);
506 id->flushed_updates = dma_fence_get(updates);
507 }
508 id->pd_gpu_addr = job->vm_pd_addr;
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800509 atomic64_set(&id->owner, vm->client_id);
510 job->vm_needs_flush = needs_flush;
511 if (needs_flush) {
512 dma_fence_put(id->last_flush);
513 id->last_flush = NULL;
514 }
515 job->vm_id = id - id_mgr->ids;
516 trace_amdgpu_vm_grab_id(vm, ring, job);
517out:
518 return r;
519}
520
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400521/**
522 * amdgpu_vm_grab_id - allocate the next free VMID
523 *
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400524 * @vm: vm to allocate id for
Christian König7f8a5292015-07-20 16:09:40 +0200525 * @ring: ring we want to submit job to
526 * @sync: sync object where we add dependencies
Christian König94dd0a42016-01-18 17:01:42 +0100527 * @fence: fence protecting ID from reuse
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400528 *
Christian König7f8a5292015-07-20 16:09:40 +0200529 * Allocate an id for the vm, adding fences to the sync obj as necessary.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400530 */
Christian König7f8a5292015-07-20 16:09:40 +0200531int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
Chris Wilsonf54d1862016-10-25 13:00:45 +0100532 struct amdgpu_sync *sync, struct dma_fence *fence,
Chunming Zhoufd53be32016-07-01 17:59:01 +0800533 struct amdgpu_job *job)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400534{
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400535 struct amdgpu_device *adev = ring->adev;
Christian König2e819842017-03-30 16:50:47 +0200536 unsigned vmhub = ring->funcs->vmhub;
Christian König76456702017-04-06 17:52:39 +0200537 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
Christian König090b7672016-07-08 10:21:02 +0200538 uint64_t fence_context = adev->fence_context + ring->idx;
Chris Wilsonf54d1862016-10-25 13:00:45 +0100539 struct dma_fence *updates = sync->last_vm_update;
Christian König8d76001e2016-05-23 16:00:32 +0200540 struct amdgpu_vm_id *id, *idle;
Chris Wilsonf54d1862016-10-25 13:00:45 +0100541 struct dma_fence **fences;
Christian König1fbb2e92016-06-01 10:47:36 +0200542 unsigned i;
543 int r = 0;
544
Christian König76456702017-04-06 17:52:39 +0200545 mutex_lock(&id_mgr->lock);
Chunming Zhou7a63eb22017-04-21 11:13:56 +0800546 if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
547 r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
548 mutex_unlock(&id_mgr->lock);
549 return r;
550 }
551 fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
552 if (!fences) {
553 mutex_unlock(&id_mgr->lock);
554 return -ENOMEM;
555 }
Christian König36fd7c52016-05-23 15:30:08 +0200556 /* Check if we have an idle VMID */
Christian König1fbb2e92016-06-01 10:47:36 +0200557 i = 0;
Christian König76456702017-04-06 17:52:39 +0200558 list_for_each_entry(idle, &id_mgr->ids_lru, list) {
Christian König1fbb2e92016-06-01 10:47:36 +0200559 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
560 if (!fences[i])
Christian König36fd7c52016-05-23 15:30:08 +0200561 break;
Christian König1fbb2e92016-06-01 10:47:36 +0200562 ++i;
Christian König36fd7c52016-05-23 15:30:08 +0200563 }
Christian Königbcb1ba32016-03-08 15:40:11 +0100564
Christian König1fbb2e92016-06-01 10:47:36 +0200565 /* If we can't find a idle VMID to use, wait till one becomes available */
Christian König76456702017-04-06 17:52:39 +0200566 if (&idle->list == &id_mgr->ids_lru) {
Christian König1fbb2e92016-06-01 10:47:36 +0200567 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
568 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
Chris Wilsonf54d1862016-10-25 13:00:45 +0100569 struct dma_fence_array *array;
Christian König1fbb2e92016-06-01 10:47:36 +0200570 unsigned j;
Christian König8d76001e2016-05-23 16:00:32 +0200571
Christian König1fbb2e92016-06-01 10:47:36 +0200572 for (j = 0; j < i; ++j)
Chris Wilsonf54d1862016-10-25 13:00:45 +0100573 dma_fence_get(fences[j]);
Christian König8d76001e2016-05-23 16:00:32 +0200574
Chris Wilsonf54d1862016-10-25 13:00:45 +0100575 array = dma_fence_array_create(i, fences, fence_context,
Christian König1fbb2e92016-06-01 10:47:36 +0200576 seqno, true);
577 if (!array) {
578 for (j = 0; j < i; ++j)
Chris Wilsonf54d1862016-10-25 13:00:45 +0100579 dma_fence_put(fences[j]);
Christian König1fbb2e92016-06-01 10:47:36 +0200580 kfree(fences);
581 r = -ENOMEM;
582 goto error;
583 }
Christian König8d76001e2016-05-23 16:00:32 +0200584
Christian König8d76001e2016-05-23 16:00:32 +0200585
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -0500586 r = amdgpu_sync_fence(ring->adev, sync, &array->base, false);
Chris Wilsonf54d1862016-10-25 13:00:45 +0100587 dma_fence_put(&array->base);
Christian König1fbb2e92016-06-01 10:47:36 +0200588 if (r)
589 goto error;
Christian König8d76001e2016-05-23 16:00:32 +0200590
Christian König76456702017-04-06 17:52:39 +0200591 mutex_unlock(&id_mgr->lock);
Christian König1fbb2e92016-06-01 10:47:36 +0200592 return 0;
Christian König8d76001e2016-05-23 16:00:32 +0200593
Christian König1fbb2e92016-06-01 10:47:36 +0200594 }
595 kfree(fences);
Christian König8d76001e2016-05-23 16:00:32 +0200596
Christian König6f1ceab2017-07-11 16:59:21 +0200597 job->vm_needs_flush = vm->use_cpu_for_update;
Christian König1fbb2e92016-06-01 10:47:36 +0200598 /* Check if we can use a VMID already assigned to this VM */
Christian König76456702017-04-06 17:52:39 +0200599 list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100600 struct dma_fence *flushed;
Christian König6f1ceab2017-07-11 16:59:21 +0200601 bool needs_flush = vm->use_cpu_for_update;
Christian König8d76001e2016-05-23 16:00:32 +0200602
Christian König1fbb2e92016-06-01 10:47:36 +0200603 /* Check all the prerequisites to using this VMID */
Christian König641e9402017-04-03 13:59:25 +0200604 if (amdgpu_vm_had_gpu_reset(adev, id))
Chunming Zhou6adb0512016-06-27 17:06:01 +0800605 continue;
Christian König1fbb2e92016-06-01 10:47:36 +0200606
607 if (atomic64_read(&id->owner) != vm->client_id)
608 continue;
609
Chunming Zhoufd53be32016-07-01 17:59:01 +0800610 if (job->vm_pd_addr != id->pd_gpu_addr)
Christian König1fbb2e92016-06-01 10:47:36 +0200611 continue;
612
Christian König87c910d2017-03-30 16:56:20 +0200613 if (!id->last_flush ||
614 (id->last_flush->context != fence_context &&
615 !dma_fence_is_signaled(id->last_flush)))
616 needs_flush = true;
Christian König1fbb2e92016-06-01 10:47:36 +0200617
618 flushed = id->flushed_updates;
Christian König87c910d2017-03-30 16:56:20 +0200619 if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
620 needs_flush = true;
621
622 /* Concurrent flushes are only possible starting with Vega10 */
623 if (adev->asic_type < CHIP_VEGA10 && needs_flush)
Christian König1fbb2e92016-06-01 10:47:36 +0200624 continue;
625
Christian König3dab83b2016-06-01 13:31:17 +0200626 /* Good we can use this VMID. Remember this submission as
627 * user of the VMID.
628 */
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -0500629 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
Christian König1fbb2e92016-06-01 10:47:36 +0200630 if (r)
631 goto error;
Christian König8d76001e2016-05-23 16:00:32 +0200632
Christian König87c910d2017-03-30 16:56:20 +0200633 if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
634 dma_fence_put(id->flushed_updates);
635 id->flushed_updates = dma_fence_get(updates);
636 }
Christian König8d76001e2016-05-23 16:00:32 +0200637
Christian König87c910d2017-03-30 16:56:20 +0200638 if (needs_flush)
639 goto needs_flush;
640 else
641 goto no_flush_needed;
Christian König8d76001e2016-05-23 16:00:32 +0200642
Christian König4f618e72017-04-06 15:18:21 +0200643 };
Chunming Zhou8e9fbeb2016-03-17 11:41:37 +0800644
Christian König1fbb2e92016-06-01 10:47:36 +0200645 /* Still no ID to use? Then use the idle one found earlier */
646 id = idle;
647
648 /* Remember this submission as user of the VMID */
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -0500649 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
Christian König832a9022016-02-15 12:33:02 +0100650 if (r)
651 goto error;
Christian König4ff37a82016-02-26 16:18:26 +0100652
Christian König87c910d2017-03-30 16:56:20 +0200653 id->pd_gpu_addr = job->vm_pd_addr;
654 dma_fence_put(id->flushed_updates);
655 id->flushed_updates = dma_fence_get(updates);
Christian König87c910d2017-03-30 16:56:20 +0200656 atomic64_set(&id->owner, vm->client_id);
657
658needs_flush:
659 job->vm_needs_flush = true;
Chris Wilsonf54d1862016-10-25 13:00:45 +0100660 dma_fence_put(id->last_flush);
Christian König41d9eb22016-03-01 16:46:18 +0100661 id->last_flush = NULL;
662
Christian König87c910d2017-03-30 16:56:20 +0200663no_flush_needed:
Christian König76456702017-04-06 17:52:39 +0200664 list_move_tail(&id->list, &id_mgr->ids_lru);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400665
Christian König76456702017-04-06 17:52:39 +0200666 job->vm_id = id - id_mgr->ids;
Christian Königc5296d12017-04-07 15:31:13 +0200667 trace_amdgpu_vm_grab_id(vm, ring, job);
Christian König832a9022016-02-15 12:33:02 +0100668
669error:
Christian König76456702017-04-06 17:52:39 +0200670 mutex_unlock(&id_mgr->lock);
Christian Königa9a78b32016-01-21 10:19:11 +0100671 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400672}
673
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800674static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
675 struct amdgpu_vm *vm,
676 unsigned vmhub)
Alex Deucher93dcc372016-06-17 17:05:15 -0400677{
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800678 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
Alex Deucher93dcc372016-06-17 17:05:15 -0400679
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800680 mutex_lock(&id_mgr->lock);
681 if (vm->reserved_vmid[vmhub]) {
682 list_add(&vm->reserved_vmid[vmhub]->list,
683 &id_mgr->ids_lru);
684 vm->reserved_vmid[vmhub] = NULL;
Chunming Zhouc3505772017-04-21 15:51:04 +0800685 atomic_dec(&id_mgr->reserved_vmid_num);
Alex Deucher93dcc372016-06-17 17:05:15 -0400686 }
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800687 mutex_unlock(&id_mgr->lock);
Alex Deucher93dcc372016-06-17 17:05:15 -0400688}
689
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800690static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
691 struct amdgpu_vm *vm,
692 unsigned vmhub)
Alex Xiee60f8db2017-03-09 11:36:26 -0500693{
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800694 struct amdgpu_vm_id_manager *id_mgr;
695 struct amdgpu_vm_id *idle;
696 int r = 0;
Alex Xiee60f8db2017-03-09 11:36:26 -0500697
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800698 id_mgr = &adev->vm_manager.id_mgr[vmhub];
699 mutex_lock(&id_mgr->lock);
700 if (vm->reserved_vmid[vmhub])
701 goto unlock;
Chunming Zhouc3505772017-04-21 15:51:04 +0800702 if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
703 AMDGPU_VM_MAX_RESERVED_VMID) {
704 DRM_ERROR("Over limitation of reserved vmid\n");
705 atomic_dec(&id_mgr->reserved_vmid_num);
706 r = -EINVAL;
707 goto unlock;
708 }
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800709 /* Select the first entry VMID */
710 idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
711 list_del_init(&idle->list);
712 vm->reserved_vmid[vmhub] = idle;
713 mutex_unlock(&id_mgr->lock);
Alex Xiee60f8db2017-03-09 11:36:26 -0500714
Chunming Zhou1e9ef262017-04-20 16:18:48 +0800715 return 0;
716unlock:
717 mutex_unlock(&id_mgr->lock);
718 return r;
719}
720
Alex Xiee59c0202017-06-01 09:42:59 -0400721/**
722 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
723 *
724 * @adev: amdgpu_device pointer
725 */
726void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
727{
728 const struct amdgpu_ip_block *ip_block;
729 bool has_compute_vm_bug;
730 struct amdgpu_ring *ring;
731 int i;
732
733 has_compute_vm_bug = false;
734
735 ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
736 if (ip_block) {
737 /* Compute has a VM bug for GFX version < 7.
738 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
739 if (ip_block->version->major <= 7)
740 has_compute_vm_bug = true;
741 else if (ip_block->version->major == 8)
742 if (adev->gfx.mec_fw_version < 673)
743 has_compute_vm_bug = true;
744 }
745
746 for (i = 0; i < adev->num_rings; i++) {
747 ring = adev->rings[i];
748 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
749 /* only compute rings */
750 ring->has_compute_vm_bug = has_compute_vm_bug;
751 else
752 ring->has_compute_vm_bug = false;
753 }
754}
755
Chunming Zhoub9bf33d2017-05-11 14:52:48 -0400756bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
757 struct amdgpu_job *job)
758{
759 struct amdgpu_device *adev = ring->adev;
760 unsigned vmhub = ring->funcs->vmhub;
761 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
762 struct amdgpu_vm_id *id;
763 bool gds_switch_needed;
Alex Xiee59c0202017-06-01 09:42:59 -0400764 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
Chunming Zhoub9bf33d2017-05-11 14:52:48 -0400765
766 if (job->vm_id == 0)
767 return false;
768 id = &id_mgr->ids[job->vm_id];
769 gds_switch_needed = ring->funcs->emit_gds_switch && (
770 id->gds_base != job->gds_base ||
771 id->gds_size != job->gds_size ||
772 id->gws_base != job->gws_base ||
773 id->gws_size != job->gws_size ||
774 id->oa_base != job->oa_base ||
775 id->oa_size != job->oa_size);
776
777 if (amdgpu_vm_had_gpu_reset(adev, id))
778 return true;
Alex Xiebb37b672017-05-30 23:50:10 -0400779
780 return vm_flush_needed || gds_switch_needed;
Chunming Zhoub9bf33d2017-05-11 14:52:48 -0400781}
782
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -0400783static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
784{
785 return (adev->mc.real_vram_size == adev->mc.visible_vram_size);
Alex Xiee60f8db2017-03-09 11:36:26 -0500786}
787
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400788/**
789 * amdgpu_vm_flush - hardware flush the vm
790 *
791 * @ring: ring to use for flush
Christian Königcffadc82016-03-01 13:34:49 +0100792 * @vm_id: vmid number to use
Christian König4ff37a82016-02-26 16:18:26 +0100793 * @pd_addr: address of the page directory
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400794 *
Christian König4ff37a82016-02-26 16:18:26 +0100795 * Emit a VM flush when it is necessary.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400796 */
Monk Liu8fdf0742017-06-06 17:25:13 +0800797int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400798{
Christian König971fe9a92016-03-01 15:09:25 +0100799 struct amdgpu_device *adev = ring->adev;
Christian König76456702017-04-06 17:52:39 +0200800 unsigned vmhub = ring->funcs->vmhub;
801 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
802 struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
Christian Königd564a062016-03-01 15:51:53 +0100803 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
Chunming Zhoufd53be32016-07-01 17:59:01 +0800804 id->gds_base != job->gds_base ||
805 id->gds_size != job->gds_size ||
806 id->gws_base != job->gws_base ||
807 id->gws_size != job->gws_size ||
808 id->oa_base != job->oa_base ||
809 id->oa_size != job->oa_size);
Flora Cuide37e682017-05-18 13:56:22 +0800810 bool vm_flush_needed = job->vm_needs_flush;
Christian Königc0e51932017-04-03 14:16:07 +0200811 unsigned patch_offset = 0;
Christian König41d9eb22016-03-01 16:46:18 +0100812 int r;
Christian Königd564a062016-03-01 15:51:53 +0100813
Christian Königf7d015b2017-04-03 14:28:26 +0200814 if (amdgpu_vm_had_gpu_reset(adev, id)) {
815 gds_switch_needed = true;
816 vm_flush_needed = true;
817 }
Christian König971fe9a92016-03-01 15:09:25 +0100818
Monk Liu8fdf0742017-06-06 17:25:13 +0800819 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
Christian Königf7d015b2017-04-03 14:28:26 +0200820 return 0;
Christian König41d9eb22016-03-01 16:46:18 +0100821
Christian Königc0e51932017-04-03 14:16:07 +0200822 if (ring->funcs->init_cond_exec)
823 patch_offset = amdgpu_ring_init_cond_exec(ring);
Christian König41d9eb22016-03-01 16:46:18 +0100824
Monk Liu8fdf0742017-06-06 17:25:13 +0800825 if (need_pipe_sync)
826 amdgpu_ring_emit_pipeline_sync(ring);
827
Christian Königf7d015b2017-04-03 14:28:26 +0200828 if (ring->funcs->emit_vm_flush && vm_flush_needed) {
Christian Königc0e51932017-04-03 14:16:07 +0200829 struct dma_fence *fence;
Monk Liue9d672b2017-03-15 12:18:57 +0800830
Christian König9a94f5a2017-05-12 14:46:23 +0200831 trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
832 amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
Monk Liue9d672b2017-03-15 12:18:57 +0800833
Christian Königc0e51932017-04-03 14:16:07 +0200834 r = amdgpu_fence_emit(ring, &fence);
835 if (r)
836 return r;
Monk Liue9d672b2017-03-15 12:18:57 +0800837
Christian König76456702017-04-06 17:52:39 +0200838 mutex_lock(&id_mgr->lock);
Christian Königc0e51932017-04-03 14:16:07 +0200839 dma_fence_put(id->last_flush);
840 id->last_flush = fence;
Chunming Zhoubea396722017-05-10 13:02:39 +0800841 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
Christian König76456702017-04-06 17:52:39 +0200842 mutex_unlock(&id_mgr->lock);
Christian Königc0e51932017-04-03 14:16:07 +0200843 }
Monk Liue9d672b2017-03-15 12:18:57 +0800844
Chunming Zhou7c4378f2017-05-11 18:22:17 +0800845 if (ring->funcs->emit_gds_switch && gds_switch_needed) {
Christian Königc0e51932017-04-03 14:16:07 +0200846 id->gds_base = job->gds_base;
847 id->gds_size = job->gds_size;
848 id->gws_base = job->gws_base;
849 id->gws_size = job->gws_size;
850 id->oa_base = job->oa_base;
851 id->oa_size = job->oa_size;
852 amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
853 job->gds_size, job->gws_base,
854 job->gws_size, job->oa_base,
855 job->oa_size);
856 }
857
858 if (ring->funcs->patch_cond_exec)
859 amdgpu_ring_patch_cond_exec(ring, patch_offset);
860
861 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
862 if (ring->funcs->emit_switch_buffer) {
863 amdgpu_ring_emit_switch_buffer(ring);
864 amdgpu_ring_emit_switch_buffer(ring);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400865 }
Christian König41d9eb22016-03-01 16:46:18 +0100866 return 0;
Christian König971fe9a92016-03-01 15:09:25 +0100867}
868
869/**
870 * amdgpu_vm_reset_id - reset VMID to zero
871 *
872 * @adev: amdgpu device structure
873 * @vm_id: vmid number to use
874 *
875 * Reset saved GDW, GWS and OA to force switch on next flush.
876 */
Christian König76456702017-04-06 17:52:39 +0200877void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
878 unsigned vmid)
Christian König971fe9a92016-03-01 15:09:25 +0100879{
Christian König76456702017-04-06 17:52:39 +0200880 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
881 struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
Christian König971fe9a92016-03-01 15:09:25 +0100882
Christian Königb3c85a02017-05-10 20:06:58 +0200883 atomic64_set(&id->owner, 0);
Christian Königbcb1ba32016-03-08 15:40:11 +0100884 id->gds_base = 0;
885 id->gds_size = 0;
886 id->gws_base = 0;
887 id->gws_size = 0;
888 id->oa_base = 0;
889 id->oa_size = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400890}
891
892/**
Christian Königb3c85a02017-05-10 20:06:58 +0200893 * amdgpu_vm_reset_all_id - reset VMID to zero
894 *
895 * @adev: amdgpu device structure
896 *
897 * Reset VMID to force flush on next use
898 */
899void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
900{
901 unsigned i, j;
902
903 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
904 struct amdgpu_vm_id_manager *id_mgr =
905 &adev->vm_manager.id_mgr[i];
906
907 for (j = 1; j < id_mgr->num_ids; ++j)
908 amdgpu_vm_reset_id(adev, i, j);
909 }
910}
911
912/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400913 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
914 *
915 * @vm: requested vm
916 * @bo: requested buffer object
917 *
Christian König8843dbb2016-01-26 12:17:11 +0100918 * Find @bo inside the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400919 * Search inside the @bos vm list for the requested vm
920 * Returns the found bo_va or NULL if none is found
921 *
922 * Object has to be reserved!
923 */
924struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
925 struct amdgpu_bo *bo)
926{
927 struct amdgpu_bo_va *bo_va;
928
Christian Königec681542017-08-01 10:51:43 +0200929 list_for_each_entry(bo_va, &bo->va, base.bo_list) {
930 if (bo_va->base.vm == vm) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400931 return bo_va;
932 }
933 }
934 return NULL;
935}
936
937/**
Christian Königafef8b82016-08-12 13:29:18 +0200938 * amdgpu_vm_do_set_ptes - helper to call the right asic function
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400939 *
Christian König29efc4f2016-08-04 14:52:50 +0200940 * @params: see amdgpu_pte_update_params definition
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400941 * @pe: addr of the page entry
942 * @addr: dst addr to write into pe
943 * @count: number of page entries to update
944 * @incr: increase next addr by incr bytes
945 * @flags: hw access flags
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400946 *
947 * Traces the parameters and calls the right asic functions
948 * to setup the page table using the DMA.
949 */
Christian Königafef8b82016-08-12 13:29:18 +0200950static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
951 uint64_t pe, uint64_t addr,
952 unsigned count, uint32_t incr,
Chunming Zhou6b777602016-09-21 16:19:19 +0800953 uint64_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400954{
Christian Königec2f05f2016-09-25 16:11:52 +0200955 trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400956
Christian Königafef8b82016-08-12 13:29:18 +0200957 if (count < 3) {
Christian Königde9ea7b2016-08-12 11:33:30 +0200958 amdgpu_vm_write_pte(params->adev, params->ib, pe,
959 addr | flags, count, incr);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400960
961 } else {
Christian König27c5f362016-08-04 15:02:49 +0200962 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400963 count, incr, flags);
964 }
965}
966
967/**
Christian Königafef8b82016-08-12 13:29:18 +0200968 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
969 *
970 * @params: see amdgpu_pte_update_params definition
971 * @pe: addr of the page entry
972 * @addr: dst addr to write into pe
973 * @count: number of page entries to update
974 * @incr: increase next addr by incr bytes
975 * @flags: hw access flags
976 *
977 * Traces the parameters and calls the DMA function to copy the PTEs.
978 */
979static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
980 uint64_t pe, uint64_t addr,
981 unsigned count, uint32_t incr,
Chunming Zhou6b777602016-09-21 16:19:19 +0800982 uint64_t flags)
Christian Königafef8b82016-08-12 13:29:18 +0200983{
Christian Königec2f05f2016-09-25 16:11:52 +0200984 uint64_t src = (params->src + (addr >> 12) * 8);
Christian Königafef8b82016-08-12 13:29:18 +0200985
Christian Königec2f05f2016-09-25 16:11:52 +0200986
987 trace_amdgpu_vm_copy_ptes(pe, src, count);
988
989 amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
Christian Königafef8b82016-08-12 13:29:18 +0200990}
991
992/**
Christian Königb07c9d22015-11-30 13:26:07 +0100993 * amdgpu_vm_map_gart - Resolve gart mapping of addr
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400994 *
Christian Königb07c9d22015-11-30 13:26:07 +0100995 * @pages_addr: optional DMA address to use for lookup
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400996 * @addr: the unmapped addr
997 *
998 * Look up the physical address of the page that the pte resolves
Christian Königb07c9d22015-11-30 13:26:07 +0100999 * to and return the pointer for the page table entry.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001000 */
Christian Königde9ea7b2016-08-12 11:33:30 +02001001static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001002{
1003 uint64_t result;
1004
Christian Königde9ea7b2016-08-12 11:33:30 +02001005 /* page table offset */
1006 result = pages_addr[addr >> PAGE_SHIFT];
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001007
Christian Königde9ea7b2016-08-12 11:33:30 +02001008 /* in case cpu page size != gpu page size*/
1009 result |= addr & (~PAGE_MASK);
Christian Königb07c9d22015-11-30 13:26:07 +01001010
1011 result &= 0xFFFFFFFFFFFFF000ULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001012
1013 return result;
1014}
1015
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001016/**
1017 * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
1018 *
1019 * @params: see amdgpu_pte_update_params definition
1020 * @pe: kmap addr of the page entry
1021 * @addr: dst addr to write into pe
1022 * @count: number of page entries to update
1023 * @incr: increase next addr by incr bytes
1024 * @flags: hw access flags
1025 *
1026 * Write count number of PT/PD entries directly.
1027 */
1028static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
1029 uint64_t pe, uint64_t addr,
1030 unsigned count, uint32_t incr,
1031 uint64_t flags)
1032{
1033 unsigned int i;
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001034 uint64_t value;
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001035
Christian König03918b32017-07-11 17:15:37 +02001036 trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
1037
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001038 for (i = 0; i < count; i++) {
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001039 value = params->pages_addr ?
1040 amdgpu_vm_map_gart(params->pages_addr, addr) :
1041 addr;
Harish Kasiviswanathana19240052017-06-09 17:47:28 -04001042 amdgpu_gart_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001043 i, value, flags);
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001044 addr += incr;
1045 }
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001046}
1047
Christian Königa33cab72017-07-11 17:13:00 +02001048static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1049 void *owner)
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001050{
1051 struct amdgpu_sync sync;
1052 int r;
1053
1054 amdgpu_sync_create(&sync);
Andres Rodriguez177ae092017-09-15 20:44:06 -04001055 amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001056 r = amdgpu_sync_wait(&sync, true);
1057 amdgpu_sync_free(&sync);
1058
1059 return r;
1060}
1061
Christian Königf8991ba2016-09-16 15:36:49 +02001062/*
Christian König194d2162016-10-12 15:13:52 +02001063 * amdgpu_vm_update_level - update a single level in the hierarchy
Christian Königf8991ba2016-09-16 15:36:49 +02001064 *
1065 * @adev: amdgpu_device pointer
1066 * @vm: requested vm
Christian König194d2162016-10-12 15:13:52 +02001067 * @parent: parent directory
Christian Königf8991ba2016-09-16 15:36:49 +02001068 *
Christian König194d2162016-10-12 15:13:52 +02001069 * Makes sure all entries in @parent are up to date.
Christian Königf8991ba2016-09-16 15:36:49 +02001070 * Returns 0 for success, error for failure.
1071 */
Christian König194d2162016-10-12 15:13:52 +02001072static int amdgpu_vm_update_level(struct amdgpu_device *adev,
1073 struct amdgpu_vm *vm,
Christian Königea097292017-08-09 14:15:46 +02001074 struct amdgpu_vm_pt *parent)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001075{
Christian Königf8991ba2016-09-16 15:36:49 +02001076 struct amdgpu_bo *shadow;
Harish Kasiviswanathana19240052017-06-09 17:47:28 -04001077 struct amdgpu_ring *ring = NULL;
1078 uint64_t pd_addr, shadow_addr = 0;
Christian König94c6f5e2017-11-30 14:12:53 +01001079 unsigned pt_idx, ndw = 0;
Christian Königd71518b2016-02-01 12:20:25 +01001080 struct amdgpu_job *job;
Christian König29efc4f2016-08-04 14:52:50 +02001081 struct amdgpu_pte_update_params params;
Dave Airlie220196b2016-10-28 11:33:52 +10001082 struct dma_fence *fence = NULL;
Christian Königea097292017-08-09 14:15:46 +02001083 uint32_t incr;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +08001084
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001085 int r;
1086
Christian König194d2162016-10-12 15:13:52 +02001087 if (!parent->entries)
1088 return 0;
Christian Königd71518b2016-02-01 12:20:25 +01001089
Christian König27c5f362016-08-04 15:02:49 +02001090 memset(&params, 0, sizeof(params));
1091 params.adev = adev;
Christian König3f3333f2017-08-03 14:02:13 +02001092 shadow = parent->base.bo->shadow;
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001093
Alex Deucher69277982017-07-13 15:37:11 -04001094 if (vm->use_cpu_for_update) {
Christian König3f3333f2017-08-03 14:02:13 +02001095 pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
Christian Königa33cab72017-07-11 17:13:00 +02001096 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
Christian König0a096fb2017-07-12 10:01:48 +02001097 if (unlikely(r))
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001098 return r;
Christian König0a096fb2017-07-12 10:01:48 +02001099
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001100 params.func = amdgpu_vm_cpu_set_ptes;
1101 } else {
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001102 ring = container_of(vm->entity.sched, struct amdgpu_ring,
1103 sched);
1104
1105 /* padding, etc. */
1106 ndw = 64;
1107
1108 /* assume the worst case */
1109 ndw += parent->last_entry_used * 6;
1110
Christian König3f3333f2017-08-03 14:02:13 +02001111 pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04001112
1113 if (shadow) {
1114 shadow_addr = amdgpu_bo_gpu_offset(shadow);
1115 ndw *= 2;
1116 } else {
1117 shadow_addr = 0;
1118 }
1119
1120 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1121 if (r)
1122 return r;
1123
1124 params.ib = &job->ibs[0];
1125 params.func = amdgpu_vm_do_set_ptes;
1126 }
1127
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001128
Christian König194d2162016-10-12 15:13:52 +02001129 /* walk over the address space and update the directory */
1130 for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
Christian Königea097292017-08-09 14:15:46 +02001131 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
1132 struct amdgpu_bo *bo = entry->base.bo;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001133 uint64_t pde, pt;
1134
1135 if (bo == NULL)
1136 continue;
1137
Christian Königea097292017-08-09 14:15:46 +02001138 spin_lock(&vm->status_lock);
1139 list_del_init(&entry->base.vm_status);
1140 spin_unlock(&vm->status_lock);
1141
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001142 pt = amdgpu_bo_gpu_offset(bo);
Christian König53e2e912017-05-15 15:19:10 +02001143 pt = amdgpu_gart_get_vm_pde(adev, pt);
Christian König4ab40162017-08-03 20:30:50 +02001144 /* Don't update huge pages here */
1145 if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
1146 parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
Christian Königf8991ba2016-09-16 15:36:49 +02001147 continue;
1148
Christian König4ab40162017-08-03 20:30:50 +02001149 parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001150
Christian Königea097292017-08-09 14:15:46 +02001151 incr = amdgpu_bo_size(bo);
Christian König94c6f5e2017-11-30 14:12:53 +01001152 if (shadow) {
1153 pde = shadow_addr + pt_idx * 8;
1154 params.func(&params, pde, pt, 1, incr,
1155 AMDGPU_PTE_VALID);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001156 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001157
Christian König94c6f5e2017-11-30 14:12:53 +01001158 pde = pd_addr + pt_idx * 8;
1159 params.func(&params, pde, pt, 1, incr, AMDGPU_PTE_VALID);
Chunming Zhoud5fc5e82015-07-21 16:52:10 +08001160 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001161
Christian König0a096fb2017-07-12 10:01:48 +02001162 if (!vm->use_cpu_for_update) {
1163 if (params.ib->length_dw == 0) {
1164 amdgpu_job_free(job);
1165 } else {
1166 amdgpu_ring_pad_ib(ring, params.ib);
Christian König3f3333f2017-08-03 14:02:13 +02001167 amdgpu_sync_resv(adev, &job->sync,
1168 parent->base.bo->tbo.resv,
Andres Rodriguez177ae092017-09-15 20:44:06 -04001169 AMDGPU_FENCE_OWNER_VM, false);
Christian König0a096fb2017-07-12 10:01:48 +02001170 if (shadow)
1171 amdgpu_sync_resv(adev, &job->sync,
1172 shadow->tbo.resv,
Andres Rodriguez177ae092017-09-15 20:44:06 -04001173 AMDGPU_FENCE_OWNER_VM, false);
Christian Königf8991ba2016-09-16 15:36:49 +02001174
Christian König0a096fb2017-07-12 10:01:48 +02001175 WARN_ON(params.ib->length_dw > ndw);
1176 r = amdgpu_job_submit(job, ring, &vm->entity,
1177 AMDGPU_FENCE_OWNER_VM, &fence);
1178 if (r)
1179 goto error_free;
Christian Königf8991ba2016-09-16 15:36:49 +02001180
Christian König3f3333f2017-08-03 14:02:13 +02001181 amdgpu_bo_fence(parent->base.bo, fence, true);
Christian Königd5884512017-09-08 14:09:41 +02001182 dma_fence_put(vm->last_update);
1183 vm->last_update = fence;
Christian König0a096fb2017-07-12 10:01:48 +02001184 }
Christian König194d2162016-10-12 15:13:52 +02001185 }
Christian Königf8991ba2016-09-16 15:36:49 +02001186
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001187 return 0;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +08001188
1189error_free:
Christian Königd71518b2016-02-01 12:20:25 +01001190 amdgpu_job_free(job);
Chunming Zhou4af9f072015-08-03 12:57:31 +08001191 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001192}
1193
Christian König194d2162016-10-12 15:13:52 +02001194/*
Christian König92456b92017-05-12 16:09:26 +02001195 * amdgpu_vm_invalidate_level - mark all PD levels as invalid
1196 *
1197 * @parent: parent PD
1198 *
1199 * Mark all PD level as invalid after an error.
1200 */
Christian Königea097292017-08-09 14:15:46 +02001201static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
1202 struct amdgpu_vm_pt *parent)
Christian König92456b92017-05-12 16:09:26 +02001203{
1204 unsigned pt_idx;
1205
1206 /*
1207 * Recurse into the subdirectories. This recursion is harmless because
1208 * we only have a maximum of 5 layers.
1209 */
1210 for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
1211 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
1212
Christian König3f3333f2017-08-03 14:02:13 +02001213 if (!entry->base.bo)
Christian König92456b92017-05-12 16:09:26 +02001214 continue;
1215
1216 entry->addr = ~0ULL;
Christian Königea097292017-08-09 14:15:46 +02001217 spin_lock(&vm->status_lock);
Christian König481c2e92017-09-01 14:46:19 +02001218 if (list_empty(&entry->base.vm_status))
1219 list_add(&entry->base.vm_status, &vm->relocated);
Christian Königea097292017-08-09 14:15:46 +02001220 spin_unlock(&vm->status_lock);
1221 amdgpu_vm_invalidate_level(vm, entry);
Christian König92456b92017-05-12 16:09:26 +02001222 }
1223}
1224
1225/*
Christian König194d2162016-10-12 15:13:52 +02001226 * amdgpu_vm_update_directories - make sure that all directories are valid
1227 *
1228 * @adev: amdgpu_device pointer
1229 * @vm: requested vm
1230 *
1231 * Makes sure all directories are up to date.
1232 * Returns 0 for success, error for failure.
1233 */
1234int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1235 struct amdgpu_vm *vm)
1236{
Dan Carpenter78aa02c2017-09-30 11:14:13 +03001237 int r = 0;
Christian König92456b92017-05-12 16:09:26 +02001238
Christian Königea097292017-08-09 14:15:46 +02001239 spin_lock(&vm->status_lock);
1240 while (!list_empty(&vm->relocated)) {
1241 struct amdgpu_vm_bo_base *bo_base;
1242 struct amdgpu_bo *bo;
1243
1244 bo_base = list_first_entry(&vm->relocated,
1245 struct amdgpu_vm_bo_base,
1246 vm_status);
1247 spin_unlock(&vm->status_lock);
1248
1249 bo = bo_base->bo->parent;
1250 if (bo) {
1251 struct amdgpu_vm_bo_base *parent;
1252 struct amdgpu_vm_pt *pt;
1253
1254 parent = list_first_entry(&bo->va,
1255 struct amdgpu_vm_bo_base,
1256 bo_list);
1257 pt = container_of(parent, struct amdgpu_vm_pt, base);
1258
1259 r = amdgpu_vm_update_level(adev, vm, pt);
1260 if (r) {
1261 amdgpu_vm_invalidate_level(vm, &vm->root);
1262 return r;
1263 }
1264 spin_lock(&vm->status_lock);
1265 } else {
1266 spin_lock(&vm->status_lock);
1267 list_del_init(&bo_base->vm_status);
1268 }
1269 }
1270 spin_unlock(&vm->status_lock);
Christian König92456b92017-05-12 16:09:26 +02001271
Christian König68c62302017-07-11 17:23:29 +02001272 if (vm->use_cpu_for_update) {
1273 /* Flush HDP */
1274 mb();
1275 amdgpu_gart_flush_gpu_tlb(adev, 0);
1276 }
1277
Christian König92456b92017-05-12 16:09:26 +02001278 return r;
Christian König194d2162016-10-12 15:13:52 +02001279}
1280
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001281/**
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001282 * amdgpu_vm_find_entry - find the entry for an address
Christian König4e2cb642016-10-25 15:52:28 +02001283 *
1284 * @p: see amdgpu_pte_update_params definition
1285 * @addr: virtual address in question
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001286 * @entry: resulting entry or NULL
1287 * @parent: parent entry
Christian König4e2cb642016-10-25 15:52:28 +02001288 *
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001289 * Find the vm_pt entry and it's parent for the given address.
Christian König4e2cb642016-10-25 15:52:28 +02001290 */
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001291void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
1292 struct amdgpu_vm_pt **entry,
1293 struct amdgpu_vm_pt **parent)
Christian König4e2cb642016-10-25 15:52:28 +02001294{
Christian König50783142017-11-27 14:01:51 +01001295 unsigned level = 0;
Christian König4e2cb642016-10-25 15:52:28 +02001296
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001297 *parent = NULL;
1298 *entry = &p->vm->root;
1299 while ((*entry)->entries) {
Christian König50783142017-11-27 14:01:51 +01001300 unsigned idx = addr >> amdgpu_vm_level_shift(p->adev, level++);
1301
Christian König3f3333f2017-08-03 14:02:13 +02001302 idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001303 *parent = *entry;
1304 *entry = &(*entry)->entries[idx];
Christian König4e2cb642016-10-25 15:52:28 +02001305 }
1306
Christian König50783142017-11-27 14:01:51 +01001307 if (level != p->adev->vm_manager.num_level)
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001308 *entry = NULL;
1309}
Christian König4e2cb642016-10-25 15:52:28 +02001310
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001311/**
1312 * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
1313 *
1314 * @p: see amdgpu_pte_update_params definition
1315 * @entry: vm_pt entry to check
1316 * @parent: parent entry
1317 * @nptes: number of PTEs updated with this operation
1318 * @dst: destination address where the PTEs should point to
1319 * @flags: access flags fro the PTEs
1320 *
1321 * Check if we can update the PD with a huge page.
1322 */
Christian Königec5207c2017-08-03 19:24:06 +02001323static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
1324 struct amdgpu_vm_pt *entry,
1325 struct amdgpu_vm_pt *parent,
1326 unsigned nptes, uint64_t dst,
1327 uint64_t flags)
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001328{
1329 bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
1330 uint64_t pd_addr, pde;
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001331
1332 /* In the case of a mixed PT the PDE must point to it*/
1333 if (p->adev->asic_type < CHIP_VEGA10 ||
1334 nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
Felix Kuehlingb2529032017-08-17 16:37:49 -04001335 p->src ||
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001336 !(flags & AMDGPU_PTE_VALID)) {
1337
Christian König3f3333f2017-08-03 14:02:13 +02001338 dst = amdgpu_bo_gpu_offset(entry->base.bo);
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001339 dst = amdgpu_gart_get_vm_pde(p->adev, dst);
1340 flags = AMDGPU_PTE_VALID;
1341 } else {
Christian König4ab40162017-08-03 20:30:50 +02001342 /* Set the huge page flag to stop scanning at this PDE */
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001343 flags |= AMDGPU_PDE_PTE;
1344 }
1345
Christian König4ab40162017-08-03 20:30:50 +02001346 if (entry->addr == (dst | flags))
Christian Königec5207c2017-08-03 19:24:06 +02001347 return;
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001348
Christian König4ab40162017-08-03 20:30:50 +02001349 entry->addr = (dst | flags);
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001350
1351 if (use_cpu_update) {
Felix Kuehlingb2529032017-08-17 16:37:49 -04001352 /* In case a huge page is replaced with a system
1353 * memory mapping, p->pages_addr != NULL and
1354 * amdgpu_vm_cpu_set_ptes would try to translate dst
1355 * through amdgpu_vm_map_gart. But dst is already a
1356 * GPU address (of the page table). Disable
1357 * amdgpu_vm_map_gart temporarily.
1358 */
1359 dma_addr_t *tmp;
1360
1361 tmp = p->pages_addr;
1362 p->pages_addr = NULL;
1363
Christian König3f3333f2017-08-03 14:02:13 +02001364 pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001365 pde = pd_addr + (entry - parent->entries) * 8;
1366 amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
Felix Kuehlingb2529032017-08-17 16:37:49 -04001367
1368 p->pages_addr = tmp;
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001369 } else {
Christian König3f3333f2017-08-03 14:02:13 +02001370 if (parent->base.bo->shadow) {
1371 pd_addr = amdgpu_bo_gpu_offset(parent->base.bo->shadow);
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001372 pde = pd_addr + (entry - parent->entries) * 8;
1373 amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
1374 }
Christian König3f3333f2017-08-03 14:02:13 +02001375 pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001376 pde = pd_addr + (entry - parent->entries) * 8;
1377 amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
1378 }
Christian König4e2cb642016-10-25 15:52:28 +02001379}
1380
1381/**
Christian König92696dd2016-08-05 13:56:35 +02001382 * amdgpu_vm_update_ptes - make sure that page tables are valid
1383 *
1384 * @params: see amdgpu_pte_update_params definition
1385 * @vm: requested vm
1386 * @start: start of GPU address range
1387 * @end: end of GPU address range
1388 * @dst: destination address to map to, the next dst inside the function
1389 * @flags: mapping flags
1390 *
1391 * Update the page tables in the range @start - @end.
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001392 * Returns 0 for success, -EINVAL for failure.
Christian König92696dd2016-08-05 13:56:35 +02001393 */
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001394static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
Christian König92696dd2016-08-05 13:56:35 +02001395 uint64_t start, uint64_t end,
Chunming Zhou6b777602016-09-21 16:19:19 +08001396 uint64_t dst, uint64_t flags)
Christian König92696dd2016-08-05 13:56:35 +02001397{
Zhang, Jerry36b32a62017-03-29 16:08:32 +08001398 struct amdgpu_device *adev = params->adev;
1399 const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
Christian König92696dd2016-08-05 13:56:35 +02001400
Christian König301654a2017-05-16 14:30:27 +02001401 uint64_t addr, pe_start;
Christian König92696dd2016-08-05 13:56:35 +02001402 struct amdgpu_bo *pt;
Christian König301654a2017-05-16 14:30:27 +02001403 unsigned nptes;
Harish Kasiviswanathan370f0922017-06-09 17:47:27 -04001404 bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
Christian König92696dd2016-08-05 13:56:35 +02001405
1406 /* walk over the address space and update the page tables */
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001407 for (addr = start; addr < end; addr += nptes,
1408 dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
1409 struct amdgpu_vm_pt *entry, *parent;
1410
1411 amdgpu_vm_get_entry(params, addr, &entry, &parent);
1412 if (!entry)
1413 return -ENOENT;
Christian König4e2cb642016-10-25 15:52:28 +02001414
Christian König92696dd2016-08-05 13:56:35 +02001415 if ((addr & ~mask) == (end & ~mask))
1416 nptes = end - addr;
1417 else
Zhang, Jerry36b32a62017-03-29 16:08:32 +08001418 nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
Christian König92696dd2016-08-05 13:56:35 +02001419
Christian Königec5207c2017-08-03 19:24:06 +02001420 amdgpu_vm_handle_huge_pages(params, entry, parent,
1421 nptes, dst, flags);
Christian König4ab40162017-08-03 20:30:50 +02001422 /* We don't need to update PTEs for huge pages */
1423 if (entry->addr & AMDGPU_PDE_PTE)
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001424 continue;
1425
Christian König3f3333f2017-08-03 14:02:13 +02001426 pt = entry->base.bo;
Harish Kasiviswanathan370f0922017-06-09 17:47:27 -04001427 if (use_cpu_update) {
Christian Königf5e1c742017-07-20 23:45:18 +02001428 pe_start = (unsigned long)amdgpu_bo_kptr(pt);
Christian Königdd0792c2017-06-27 14:48:15 -04001429 } else {
1430 if (pt->shadow) {
1431 pe_start = amdgpu_bo_gpu_offset(pt->shadow);
1432 pe_start += (addr & mask) * 8;
1433 params->func(params, pe_start, dst, nptes,
1434 AMDGPU_GPU_PAGE_SIZE, flags);
1435 }
Harish Kasiviswanathan370f0922017-06-09 17:47:27 -04001436 pe_start = amdgpu_bo_gpu_offset(pt);
Christian Königdd0792c2017-06-27 14:48:15 -04001437 }
Christian König92696dd2016-08-05 13:56:35 +02001438
Christian König301654a2017-05-16 14:30:27 +02001439 pe_start += (addr & mask) * 8;
Christian König301654a2017-05-16 14:30:27 +02001440 params->func(params, pe_start, dst, nptes,
1441 AMDGPU_GPU_PAGE_SIZE, flags);
Christian König92696dd2016-08-05 13:56:35 +02001442 }
1443
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001444 return 0;
Christian König92696dd2016-08-05 13:56:35 +02001445}
1446
1447/*
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001448 * amdgpu_vm_frag_ptes - add fragment information to PTEs
1449 *
Christian König29efc4f2016-08-04 14:52:50 +02001450 * @params: see amdgpu_pte_update_params definition
Christian König92696dd2016-08-05 13:56:35 +02001451 * @vm: requested vm
1452 * @start: first PTE to handle
1453 * @end: last PTE to handle
1454 * @dst: addr those PTEs should point to
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001455 * @flags: hw mapping flags
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001456 * Returns 0 for success, -EINVAL for failure.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001457 */
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001458static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
Christian König92696dd2016-08-05 13:56:35 +02001459 uint64_t start, uint64_t end,
Chunming Zhou6b777602016-09-21 16:19:19 +08001460 uint64_t dst, uint64_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001461{
1462 /**
1463 * The MC L1 TLB supports variable sized pages, based on a fragment
1464 * field in the PTE. When this field is set to a non-zero value, page
1465 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1466 * flags are considered valid for all PTEs within the fragment range
1467 * and corresponding mappings are assumed to be physically contiguous.
1468 *
1469 * The L1 TLB can store a single PTE for the whole fragment,
1470 * significantly increasing the space available for translation
1471 * caching. This leads to large improvements in throughput when the
1472 * TLB is under pressure.
1473 *
1474 * The L2 TLB distributes small and large fragments into two
1475 * asymmetric partitions. The large fragment cache is significantly
1476 * larger. Thus, we try to use large fragments wherever possible.
1477 * Userspace can support this by aligning virtual base address and
1478 * allocation size to the fragment size.
1479 */
Roger He6849d472017-08-30 13:01:19 +08001480 unsigned max_frag = params->adev->vm_manager.fragment_size;
1481 int r;
Christian König31f6c1f2016-01-26 12:37:49 +01001482
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001483 /* system pages are non continuously */
Roger He6849d472017-08-30 13:01:19 +08001484 if (params->src || !(flags & AMDGPU_PTE_VALID))
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001485 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001486
Roger He6849d472017-08-30 13:01:19 +08001487 while (start != end) {
1488 uint64_t frag_flags, frag_end;
1489 unsigned frag;
1490
1491 /* This intentionally wraps around if no bit is set */
1492 frag = min((unsigned)ffs(start) - 1,
1493 (unsigned)fls64(end - start) - 1);
1494 if (frag >= max_frag) {
1495 frag_flags = AMDGPU_PTE_FRAG(max_frag);
1496 frag_end = end & ~((1ULL << max_frag) - 1);
1497 } else {
1498 frag_flags = AMDGPU_PTE_FRAG(frag);
1499 frag_end = start + (1 << frag);
1500 }
1501
1502 r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
1503 flags | frag_flags);
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001504 if (r)
1505 return r;
Roger He6849d472017-08-30 13:01:19 +08001506
1507 dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
1508 start = frag_end;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001509 }
1510
Roger He6849d472017-08-30 13:01:19 +08001511 return 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001512}
1513
1514/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001515 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1516 *
1517 * @adev: amdgpu_device pointer
Christian König3cabaa52016-06-06 10:17:58 +02001518 * @exclusive: fence we need to sync to
Christian Königfa3ab3c2016-03-18 21:00:35 +01001519 * @pages_addr: DMA addresses to use for mapping
Christian Königa14faa62016-01-25 14:27:31 +01001520 * @vm: requested vm
1521 * @start: start of mapped range
1522 * @last: last mapped entry
1523 * @flags: flags for the entries
1524 * @addr: addr to set the area to
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001525 * @fence: optional resulting fence
1526 *
Christian Königa14faa62016-01-25 14:27:31 +01001527 * Fill in the page table entries between @start and @last.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001528 * Returns 0 for success, -EINVAL for failure.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001529 */
1530static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
Chris Wilsonf54d1862016-10-25 13:00:45 +01001531 struct dma_fence *exclusive,
Christian Königfa3ab3c2016-03-18 21:00:35 +01001532 dma_addr_t *pages_addr,
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001533 struct amdgpu_vm *vm,
Christian Königa14faa62016-01-25 14:27:31 +01001534 uint64_t start, uint64_t last,
Chunming Zhou6b777602016-09-21 16:19:19 +08001535 uint64_t flags, uint64_t addr,
Chris Wilsonf54d1862016-10-25 13:00:45 +01001536 struct dma_fence **fence)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001537{
Christian König2d55e452016-02-08 17:37:38 +01001538 struct amdgpu_ring *ring;
Christian Königa1e08d32016-01-26 11:40:46 +01001539 void *owner = AMDGPU_FENCE_OWNER_VM;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001540 unsigned nptes, ncmds, ndw;
Christian Königd71518b2016-02-01 12:20:25 +01001541 struct amdgpu_job *job;
Christian König29efc4f2016-08-04 14:52:50 +02001542 struct amdgpu_pte_update_params params;
Chris Wilsonf54d1862016-10-25 13:00:45 +01001543 struct dma_fence *f = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001544 int r;
1545
Christian Königafef8b82016-08-12 13:29:18 +02001546 memset(&params, 0, sizeof(params));
1547 params.adev = adev;
Christian König49ac8a22016-10-13 15:09:08 +02001548 params.vm = vm;
Christian Königafef8b82016-08-12 13:29:18 +02001549
Christian Königa33cab72017-07-11 17:13:00 +02001550 /* sync to everything on unmapping */
1551 if (!(flags & AMDGPU_PTE_VALID))
1552 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
1553
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001554 if (vm->use_cpu_for_update) {
1555 /* params.src is used as flag to indicate system Memory */
1556 if (pages_addr)
1557 params.src = ~0;
1558
1559 /* Wait for PT BOs to be free. PTs share the same resv. object
1560 * as the root PD BO
1561 */
Christian Königa33cab72017-07-11 17:13:00 +02001562 r = amdgpu_vm_wait_pd(adev, vm, owner);
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001563 if (unlikely(r))
1564 return r;
1565
1566 params.func = amdgpu_vm_cpu_set_ptes;
1567 params.pages_addr = pages_addr;
Harish Kasiviswanathanb4d42512017-05-11 19:47:22 -04001568 return amdgpu_vm_frag_ptes(&params, start, last + 1,
1569 addr, flags);
1570 }
1571
Christian König2d55e452016-02-08 17:37:38 +01001572 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
Christian König27c5f362016-08-04 15:02:49 +02001573
Christian Königa14faa62016-01-25 14:27:31 +01001574 nptes = last - start + 1;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001575
1576 /*
Bas Nieuwenhuizen86209522017-09-07 13:23:21 +02001577 * reserve space for two commands every (1 << BLOCK_SIZE)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001578 * entries or 2k dwords (whatever is smaller)
Bas Nieuwenhuizen86209522017-09-07 13:23:21 +02001579 *
1580 * The second command is for the shadow pagetables.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001581 */
Bas Nieuwenhuizen86209522017-09-07 13:23:21 +02001582 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001583
1584 /* padding, etc. */
1585 ndw = 64;
1586
Alex Deuchercf2f0a32017-07-25 16:35:38 -04001587 /* one PDE write for each huge page */
1588 ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
1589
Christian König570144c2017-08-30 15:38:45 +02001590 if (pages_addr) {
Christian Königb0456f92016-08-11 14:06:54 +02001591 /* copy commands needed */
Yong Zhaoe6d92192017-09-19 12:58:15 -04001592 ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001593
Christian Königb0456f92016-08-11 14:06:54 +02001594 /* and also PTEs */
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001595 ndw += nptes * 2;
1596
Christian Königafef8b82016-08-12 13:29:18 +02001597 params.func = amdgpu_vm_do_copy_ptes;
1598
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001599 } else {
1600 /* set page commands needed */
Yong Zhao7bdc53f2017-09-15 18:20:37 -04001601 ndw += ncmds * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001602
Roger He6849d472017-08-30 13:01:19 +08001603 /* extra commands for begin/end fragments */
Yong Zhao7bdc53f2017-09-15 18:20:37 -04001604 ndw += 2 * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw
1605 * adev->vm_manager.fragment_size;
Christian Königafef8b82016-08-12 13:29:18 +02001606
1607 params.func = amdgpu_vm_do_set_ptes;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001608 }
1609
Christian Königd71518b2016-02-01 12:20:25 +01001610 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1611 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001612 return r;
Christian Königd71518b2016-02-01 12:20:25 +01001613
Christian König29efc4f2016-08-04 14:52:50 +02001614 params.ib = &job->ibs[0];
Chunming Zhoud5fc5e82015-07-21 16:52:10 +08001615
Christian König570144c2017-08-30 15:38:45 +02001616 if (pages_addr) {
Christian Königb0456f92016-08-11 14:06:54 +02001617 uint64_t *pte;
1618 unsigned i;
1619
1620 /* Put the PTEs at the end of the IB. */
1621 i = ndw - nptes * 2;
1622 pte= (uint64_t *)&(job->ibs->ptr[i]);
1623 params.src = job->ibs->gpu_addr + i * 4;
1624
1625 for (i = 0; i < nptes; ++i) {
1626 pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1627 AMDGPU_GPU_PAGE_SIZE);
1628 pte[i] |= flags;
1629 }
Christian Königd7a4ac62016-09-25 11:54:00 +02001630 addr = 0;
Christian Königb0456f92016-08-11 14:06:54 +02001631 }
1632
Andrey Grodzovskycebb52b2017-11-13 14:47:52 -05001633 r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
Christian König3cabaa52016-06-06 10:17:58 +02001634 if (r)
1635 goto error_free;
1636
Christian König3f3333f2017-08-03 14:02:13 +02001637 r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
Andres Rodriguez177ae092017-09-15 20:44:06 -04001638 owner, false);
Christian Königa1e08d32016-01-26 11:40:46 +01001639 if (r)
1640 goto error_free;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001641
Christian König3f3333f2017-08-03 14:02:13 +02001642 r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
Christian Königa1e08d32016-01-26 11:40:46 +01001643 if (r)
1644 goto error_free;
1645
Harish Kasiviswanathancc28c4e2017-05-11 22:39:31 -04001646 r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1647 if (r)
1648 goto error_free;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001649
Christian König29efc4f2016-08-04 14:52:50 +02001650 amdgpu_ring_pad_ib(ring, params.ib);
1651 WARN_ON(params.ib->length_dw > ndw);
Christian König2bd9ccf2016-02-01 12:53:58 +01001652 r = amdgpu_job_submit(job, ring, &vm->entity,
1653 AMDGPU_FENCE_OWNER_VM, &f);
Chunming Zhou4af9f072015-08-03 12:57:31 +08001654 if (r)
1655 goto error_free;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001656
Christian König3f3333f2017-08-03 14:02:13 +02001657 amdgpu_bo_fence(vm->root.base.bo, f, true);
Christian König284710f2017-01-30 11:09:31 +01001658 dma_fence_put(*fence);
1659 *fence = f;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001660 return 0;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +08001661
1662error_free:
Christian Königd71518b2016-02-01 12:20:25 +01001663 amdgpu_job_free(job);
Christian Königea097292017-08-09 14:15:46 +02001664 amdgpu_vm_invalidate_level(vm, &vm->root);
Chunming Zhou4af9f072015-08-03 12:57:31 +08001665 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001666}
1667
1668/**
Christian Königa14faa62016-01-25 14:27:31 +01001669 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1670 *
1671 * @adev: amdgpu_device pointer
Christian König3cabaa52016-06-06 10:17:58 +02001672 * @exclusive: fence we need to sync to
Christian König8358dce2016-03-30 10:50:25 +02001673 * @pages_addr: DMA addresses to use for mapping
Christian Königa14faa62016-01-25 14:27:31 +01001674 * @vm: requested vm
1675 * @mapping: mapped range and flags to use for the update
Christian König8358dce2016-03-30 10:50:25 +02001676 * @flags: HW flags for the mapping
Christian König63e0ba42016-08-16 17:38:37 +02001677 * @nodes: array of drm_mm_nodes with the MC addresses
Christian Königa14faa62016-01-25 14:27:31 +01001678 * @fence: optional resulting fence
1679 *
1680 * Split the mapping into smaller chunks so that each update fits
1681 * into a SDMA IB.
1682 * Returns 0 for success, -EINVAL for failure.
1683 */
1684static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
Chris Wilsonf54d1862016-10-25 13:00:45 +01001685 struct dma_fence *exclusive,
Christian König8358dce2016-03-30 10:50:25 +02001686 dma_addr_t *pages_addr,
Christian Königa14faa62016-01-25 14:27:31 +01001687 struct amdgpu_vm *vm,
1688 struct amdgpu_bo_va_mapping *mapping,
Chunming Zhou6b777602016-09-21 16:19:19 +08001689 uint64_t flags,
Christian König63e0ba42016-08-16 17:38:37 +02001690 struct drm_mm_node *nodes,
Chris Wilsonf54d1862016-10-25 13:00:45 +01001691 struct dma_fence **fence)
Christian Königa14faa62016-01-25 14:27:31 +01001692{
Christian König9fc8fc72017-09-18 13:58:30 +02001693 unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
Christian König570144c2017-08-30 15:38:45 +02001694 uint64_t pfn, start = mapping->start;
Christian Königa14faa62016-01-25 14:27:31 +01001695 int r;
1696
1697 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1698 * but in case of something, we filter the flags in first place
1699 */
1700 if (!(mapping->flags & AMDGPU_PTE_READABLE))
1701 flags &= ~AMDGPU_PTE_READABLE;
1702 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1703 flags &= ~AMDGPU_PTE_WRITEABLE;
1704
Alex Xie15b31c52017-03-03 16:47:11 -05001705 flags &= ~AMDGPU_PTE_EXECUTABLE;
1706 flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1707
Alex Xieb0fd18b2017-03-03 16:49:39 -05001708 flags &= ~AMDGPU_PTE_MTYPE_MASK;
1709 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1710
Zhang, Jerryd0766e92017-04-19 09:53:29 +08001711 if ((mapping->flags & AMDGPU_PTE_PRT) &&
1712 (adev->asic_type >= CHIP_VEGA10)) {
1713 flags |= AMDGPU_PTE_PRT;
1714 flags &= ~AMDGPU_PTE_VALID;
1715 }
1716
Christian Königa14faa62016-01-25 14:27:31 +01001717 trace_amdgpu_vm_bo_update(mapping);
1718
Christian König63e0ba42016-08-16 17:38:37 +02001719 pfn = mapping->offset >> PAGE_SHIFT;
1720 if (nodes) {
1721 while (pfn >= nodes->size) {
1722 pfn -= nodes->size;
1723 ++nodes;
1724 }
Christian Königfa3ab3c2016-03-18 21:00:35 +01001725 }
Christian Königa14faa62016-01-25 14:27:31 +01001726
Christian König63e0ba42016-08-16 17:38:37 +02001727 do {
Christian König9fc8fc72017-09-18 13:58:30 +02001728 dma_addr_t *dma_addr = NULL;
Christian König63e0ba42016-08-16 17:38:37 +02001729 uint64_t max_entries;
1730 uint64_t addr, last;
Christian Königa14faa62016-01-25 14:27:31 +01001731
Christian König63e0ba42016-08-16 17:38:37 +02001732 if (nodes) {
1733 addr = nodes->start << PAGE_SHIFT;
1734 max_entries = (nodes->size - pfn) *
1735 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1736 } else {
1737 addr = 0;
1738 max_entries = S64_MAX;
1739 }
Christian Königa14faa62016-01-25 14:27:31 +01001740
Christian König63e0ba42016-08-16 17:38:37 +02001741 if (pages_addr) {
Christian König9fc8fc72017-09-18 13:58:30 +02001742 uint64_t count;
1743
Christian König457e0fe2017-08-22 12:50:46 +02001744 max_entries = min(max_entries, 16ull * 1024ull);
Christian König9fc8fc72017-09-18 13:58:30 +02001745 for (count = 1; count < max_entries; ++count) {
1746 uint64_t idx = pfn + count;
1747
1748 if (pages_addr[idx] !=
1749 (pages_addr[idx - 1] + PAGE_SIZE))
1750 break;
1751 }
1752
1753 if (count < min_linear_pages) {
1754 addr = pfn << PAGE_SHIFT;
1755 dma_addr = pages_addr;
1756 } else {
1757 addr = pages_addr[pfn];
1758 max_entries = count;
1759 }
1760
Christian König63e0ba42016-08-16 17:38:37 +02001761 } else if (flags & AMDGPU_PTE_VALID) {
1762 addr += adev->vm_manager.vram_base_offset;
Christian König9fc8fc72017-09-18 13:58:30 +02001763 addr += pfn << PAGE_SHIFT;
Christian König63e0ba42016-08-16 17:38:37 +02001764 }
Christian König63e0ba42016-08-16 17:38:37 +02001765
Christian Königa9f87f62017-03-30 14:03:59 +02001766 last = min((uint64_t)mapping->last, start + max_entries - 1);
Christian König9fc8fc72017-09-18 13:58:30 +02001767 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
Christian Königa14faa62016-01-25 14:27:31 +01001768 start, last, flags, addr,
1769 fence);
1770 if (r)
1771 return r;
1772
Christian König63e0ba42016-08-16 17:38:37 +02001773 pfn += last - start + 1;
1774 if (nodes && nodes->size == pfn) {
1775 pfn = 0;
1776 ++nodes;
1777 }
Christian Königa14faa62016-01-25 14:27:31 +01001778 start = last + 1;
Christian König63e0ba42016-08-16 17:38:37 +02001779
Christian Königa9f87f62017-03-30 14:03:59 +02001780 } while (unlikely(start != mapping->last + 1));
Christian Königa14faa62016-01-25 14:27:31 +01001781
1782 return 0;
1783}
1784
1785/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001786 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1787 *
1788 * @adev: amdgpu_device pointer
1789 * @bo_va: requested BO and VM object
Christian König99e124f2016-08-16 14:43:17 +02001790 * @clear: if true clear the entries
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001791 *
1792 * Fill in the page table entries for @bo_va.
1793 * Returns 0 for success, -EINVAL for failure.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001794 */
1795int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1796 struct amdgpu_bo_va *bo_va,
Christian König99e124f2016-08-16 14:43:17 +02001797 bool clear)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001798{
Christian Königec681542017-08-01 10:51:43 +02001799 struct amdgpu_bo *bo = bo_va->base.bo;
1800 struct amdgpu_vm *vm = bo_va->base.vm;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001801 struct amdgpu_bo_va_mapping *mapping;
Christian König8358dce2016-03-30 10:50:25 +02001802 dma_addr_t *pages_addr = NULL;
Christian König99e124f2016-08-16 14:43:17 +02001803 struct ttm_mem_reg *mem;
Christian König63e0ba42016-08-16 17:38:37 +02001804 struct drm_mm_node *nodes;
Christian König4e55eb32017-09-11 16:54:59 +02001805 struct dma_fence *exclusive, **last_update;
Christian König457e0fe2017-08-22 12:50:46 +02001806 uint64_t flags;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001807 int r;
1808
Christian Königec681542017-08-01 10:51:43 +02001809 if (clear || !bo_va->base.bo) {
Christian König99e124f2016-08-16 14:43:17 +02001810 mem = NULL;
Christian König63e0ba42016-08-16 17:38:37 +02001811 nodes = NULL;
Christian König99e124f2016-08-16 14:43:17 +02001812 exclusive = NULL;
1813 } else {
Christian König8358dce2016-03-30 10:50:25 +02001814 struct ttm_dma_tt *ttm;
1815
Christian Königec681542017-08-01 10:51:43 +02001816 mem = &bo_va->base.bo->tbo.mem;
Christian König63e0ba42016-08-16 17:38:37 +02001817 nodes = mem->mm_node;
1818 if (mem->mem_type == TTM_PL_TT) {
Christian Königec681542017-08-01 10:51:43 +02001819 ttm = container_of(bo_va->base.bo->tbo.ttm,
1820 struct ttm_dma_tt, ttm);
Christian König8358dce2016-03-30 10:50:25 +02001821 pages_addr = ttm->dma_address;
Christian König9ab21462015-11-30 14:19:26 +01001822 }
Christian Königec681542017-08-01 10:51:43 +02001823 exclusive = reservation_object_get_excl(bo->tbo.resv);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001824 }
1825
Christian König457e0fe2017-08-22 12:50:46 +02001826 if (bo)
Christian Königec681542017-08-01 10:51:43 +02001827 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
Christian König457e0fe2017-08-22 12:50:46 +02001828 else
Christian Königa5f6b5b2017-01-30 11:01:38 +01001829 flags = 0x0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001830
Christian König4e55eb32017-09-11 16:54:59 +02001831 if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
1832 last_update = &vm->last_update;
1833 else
1834 last_update = &bo_va->last_pt_update;
1835
Christian König3d7d4d32017-08-23 16:13:33 +02001836 if (!clear && bo_va->base.moved) {
1837 bo_va->base.moved = false;
Christian König7fc11952015-07-30 11:53:42 +02001838 list_splice_init(&bo_va->valids, &bo_va->invalids);
Christian König3d7d4d32017-08-23 16:13:33 +02001839
Christian Königcb7b6ec2017-08-15 17:08:12 +02001840 } else if (bo_va->cleared != clear) {
1841 list_splice_init(&bo_va->valids, &bo_va->invalids);
Christian König3d7d4d32017-08-23 16:13:33 +02001842 }
Christian König7fc11952015-07-30 11:53:42 +02001843
1844 list_for_each_entry(mapping, &bo_va->invalids, list) {
Christian König457e0fe2017-08-22 12:50:46 +02001845 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
Christian König63e0ba42016-08-16 17:38:37 +02001846 mapping, flags, nodes,
Christian König4e55eb32017-09-11 16:54:59 +02001847 last_update);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001848 if (r)
1849 return r;
1850 }
1851
Christian König68c62302017-07-11 17:23:29 +02001852 if (vm->use_cpu_for_update) {
1853 /* Flush HDP */
1854 mb();
1855 amdgpu_gart_flush_gpu_tlb(adev, 0);
1856 }
1857
Christian Königcb7b6ec2017-08-15 17:08:12 +02001858 spin_lock(&vm->status_lock);
1859 list_del_init(&bo_va->base.vm_status);
1860 spin_unlock(&vm->status_lock);
1861
1862 list_splice_init(&bo_va->invalids, &bo_va->valids);
1863 bo_va->cleared = clear;
1864
1865 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1866 list_for_each_entry(mapping, &bo_va->valids, list)
1867 trace_amdgpu_vm_bo_mapping(mapping);
1868 }
1869
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001870 return 0;
1871}
1872
1873/**
Christian König284710f2017-01-30 11:09:31 +01001874 * amdgpu_vm_update_prt_state - update the global PRT state
1875 */
1876static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1877{
1878 unsigned long flags;
1879 bool enable;
1880
1881 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
Christian König451bc8e2017-02-14 16:02:52 +01001882 enable = !!atomic_read(&adev->vm_manager.num_prt_users);
Christian König284710f2017-01-30 11:09:31 +01001883 adev->gart.gart_funcs->set_prt(adev, enable);
1884 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1885}
1886
1887/**
Christian König4388fc22017-03-13 10:13:36 +01001888 * amdgpu_vm_prt_get - add a PRT user
Christian König451bc8e2017-02-14 16:02:52 +01001889 */
1890static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1891{
Christian König4388fc22017-03-13 10:13:36 +01001892 if (!adev->gart.gart_funcs->set_prt)
1893 return;
1894
Christian König451bc8e2017-02-14 16:02:52 +01001895 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1896 amdgpu_vm_update_prt_state(adev);
1897}
1898
1899/**
Christian König0b15f2f2017-02-14 15:47:03 +01001900 * amdgpu_vm_prt_put - drop a PRT user
1901 */
1902static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1903{
Christian König451bc8e2017-02-14 16:02:52 +01001904 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
Christian König0b15f2f2017-02-14 15:47:03 +01001905 amdgpu_vm_update_prt_state(adev);
1906}
1907
1908/**
Christian König451bc8e2017-02-14 16:02:52 +01001909 * amdgpu_vm_prt_cb - callback for updating the PRT status
Christian König284710f2017-01-30 11:09:31 +01001910 */
1911static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1912{
1913 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1914
Christian König0b15f2f2017-02-14 15:47:03 +01001915 amdgpu_vm_prt_put(cb->adev);
Christian König284710f2017-01-30 11:09:31 +01001916 kfree(cb);
1917}
1918
1919/**
Christian König451bc8e2017-02-14 16:02:52 +01001920 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1921 */
1922static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1923 struct dma_fence *fence)
1924{
Christian König4388fc22017-03-13 10:13:36 +01001925 struct amdgpu_prt_cb *cb;
Christian König451bc8e2017-02-14 16:02:52 +01001926
Christian König4388fc22017-03-13 10:13:36 +01001927 if (!adev->gart.gart_funcs->set_prt)
1928 return;
1929
1930 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
Christian König451bc8e2017-02-14 16:02:52 +01001931 if (!cb) {
1932 /* Last resort when we are OOM */
1933 if (fence)
1934 dma_fence_wait(fence, false);
1935
Dan Carpenter486a68f2017-04-03 21:41:39 +03001936 amdgpu_vm_prt_put(adev);
Christian König451bc8e2017-02-14 16:02:52 +01001937 } else {
1938 cb->adev = adev;
1939 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1940 amdgpu_vm_prt_cb))
1941 amdgpu_vm_prt_cb(fence, &cb->cb);
1942 }
1943}
1944
1945/**
Christian König284710f2017-01-30 11:09:31 +01001946 * amdgpu_vm_free_mapping - free a mapping
1947 *
1948 * @adev: amdgpu_device pointer
1949 * @vm: requested vm
1950 * @mapping: mapping to be freed
1951 * @fence: fence of the unmap operation
1952 *
1953 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1954 */
1955static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1956 struct amdgpu_vm *vm,
1957 struct amdgpu_bo_va_mapping *mapping,
1958 struct dma_fence *fence)
1959{
Christian König451bc8e2017-02-14 16:02:52 +01001960 if (mapping->flags & AMDGPU_PTE_PRT)
1961 amdgpu_vm_add_prt_cb(adev, fence);
Christian König284710f2017-01-30 11:09:31 +01001962 kfree(mapping);
1963}
1964
1965/**
Christian König451bc8e2017-02-14 16:02:52 +01001966 * amdgpu_vm_prt_fini - finish all prt mappings
1967 *
1968 * @adev: amdgpu_device pointer
1969 * @vm: requested vm
1970 *
1971 * Register a cleanup callback to disable PRT support after VM dies.
1972 */
1973static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1974{
Christian König3f3333f2017-08-03 14:02:13 +02001975 struct reservation_object *resv = vm->root.base.bo->tbo.resv;
Christian König451bc8e2017-02-14 16:02:52 +01001976 struct dma_fence *excl, **shared;
1977 unsigned i, shared_count;
1978 int r;
1979
1980 r = reservation_object_get_fences_rcu(resv, &excl,
1981 &shared_count, &shared);
1982 if (r) {
1983 /* Not enough memory to grab the fence list, as last resort
1984 * block for all the fences to complete.
1985 */
1986 reservation_object_wait_timeout_rcu(resv, true, false,
1987 MAX_SCHEDULE_TIMEOUT);
1988 return;
1989 }
1990
1991 /* Add a callback for each fence in the reservation object */
1992 amdgpu_vm_prt_get(adev);
1993 amdgpu_vm_add_prt_cb(adev, excl);
1994
1995 for (i = 0; i < shared_count; ++i) {
1996 amdgpu_vm_prt_get(adev);
1997 amdgpu_vm_add_prt_cb(adev, shared[i]);
1998 }
1999
2000 kfree(shared);
2001}
2002
2003/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002004 * amdgpu_vm_clear_freed - clear freed BOs in the PT
2005 *
2006 * @adev: amdgpu_device pointer
2007 * @vm: requested vm
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002008 * @fence: optional resulting fence (unchanged if no work needed to be done
2009 * or if an error occurred)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002010 *
2011 * Make sure all freed BOs are cleared in the PT.
2012 * Returns 0 for success.
2013 *
2014 * PTs have to be reserved and mutex must be locked!
2015 */
2016int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002017 struct amdgpu_vm *vm,
2018 struct dma_fence **fence)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002019{
2020 struct amdgpu_bo_va_mapping *mapping;
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002021 struct dma_fence *f = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002022 int r;
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002023 uint64_t init_pte_value = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002024
2025 while (!list_empty(&vm->freed)) {
2026 mapping = list_first_entry(&vm->freed,
2027 struct amdgpu_bo_va_mapping, list);
2028 list_del(&mapping->list);
Christian Könige17841b2016-03-08 17:52:01 +01002029
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002030 if (vm->pte_support_ats)
Yong Zhao6d16dac2017-08-31 15:55:00 -04002031 init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002032
Christian König570144c2017-08-30 15:38:45 +02002033 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
Christian Königfc6aa332017-04-19 14:41:19 +02002034 mapping->start, mapping->last,
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002035 init_pte_value, 0, &f);
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002036 amdgpu_vm_free_mapping(adev, vm, mapping, f);
Christian König284710f2017-01-30 11:09:31 +01002037 if (r) {
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002038 dma_fence_put(f);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002039 return r;
Christian König284710f2017-01-30 11:09:31 +01002040 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002041 }
Nicolai Hähnlef3467812017-03-23 19:36:31 +01002042
2043 if (fence && f) {
2044 dma_fence_put(*fence);
2045 *fence = f;
2046 } else {
2047 dma_fence_put(f);
2048 }
2049
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002050 return 0;
2051
2052}
2053
2054/**
Christian König73fb16e2017-08-16 11:13:48 +02002055 * amdgpu_vm_handle_moved - handle moved BOs in the PT
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002056 *
2057 * @adev: amdgpu_device pointer
2058 * @vm: requested vm
Christian König73fb16e2017-08-16 11:13:48 +02002059 * @sync: sync object to add fences to
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002060 *
Christian König73fb16e2017-08-16 11:13:48 +02002061 * Make sure all BOs which are moved are updated in the PTs.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002062 * Returns 0 for success.
2063 *
Christian König73fb16e2017-08-16 11:13:48 +02002064 * PTs have to be reserved!
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002065 */
Christian König73fb16e2017-08-16 11:13:48 +02002066int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
Christian König4e55eb32017-09-11 16:54:59 +02002067 struct amdgpu_vm *vm)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002068{
Christian König73fb16e2017-08-16 11:13:48 +02002069 bool clear;
Christian König91e1a522015-07-06 22:06:40 +02002070 int r = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002071
2072 spin_lock(&vm->status_lock);
Christian König27c7b9a2017-08-01 11:27:36 +02002073 while (!list_empty(&vm->moved)) {
Christian König4e55eb32017-09-11 16:54:59 +02002074 struct amdgpu_bo_va *bo_va;
2075
Christian König27c7b9a2017-08-01 11:27:36 +02002076 bo_va = list_first_entry(&vm->moved,
Christian Königec681542017-08-01 10:51:43 +02002077 struct amdgpu_bo_va, base.vm_status);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002078 spin_unlock(&vm->status_lock);
Christian König32b41ac2016-03-08 18:03:27 +01002079
Christian König73fb16e2017-08-16 11:13:48 +02002080 /* Per VM BOs never need to bo cleared in the page tables */
2081 clear = bo_va->base.bo->tbo.resv != vm->root.base.bo->tbo.resv;
2082
2083 r = amdgpu_vm_bo_update(adev, bo_va, clear);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002084 if (r)
2085 return r;
2086
2087 spin_lock(&vm->status_lock);
2088 }
2089 spin_unlock(&vm->status_lock);
2090
Christian König91e1a522015-07-06 22:06:40 +02002091 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002092}
2093
2094/**
2095 * amdgpu_vm_bo_add - add a bo to a specific vm
2096 *
2097 * @adev: amdgpu_device pointer
2098 * @vm: requested vm
2099 * @bo: amdgpu buffer object
2100 *
Christian König8843dbb2016-01-26 12:17:11 +01002101 * Add @bo into the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002102 * Add @bo to the list of bos associated with the vm
2103 * Returns newly added bo_va or NULL for failure
2104 *
2105 * Object has to be reserved!
2106 */
2107struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2108 struct amdgpu_vm *vm,
2109 struct amdgpu_bo *bo)
2110{
2111 struct amdgpu_bo_va *bo_va;
2112
2113 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2114 if (bo_va == NULL) {
2115 return NULL;
2116 }
Christian Königec681542017-08-01 10:51:43 +02002117 bo_va->base.vm = vm;
2118 bo_va->base.bo = bo;
2119 INIT_LIST_HEAD(&bo_va->base.bo_list);
2120 INIT_LIST_HEAD(&bo_va->base.vm_status);
2121
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002122 bo_va->ref_count = 1;
Christian König7fc11952015-07-30 11:53:42 +02002123 INIT_LIST_HEAD(&bo_va->valids);
2124 INIT_LIST_HEAD(&bo_va->invalids);
Christian König32b41ac2016-03-08 18:03:27 +01002125
Christian Königa5f6b5b2017-01-30 11:01:38 +01002126 if (bo)
Christian Königec681542017-08-01 10:51:43 +02002127 list_add_tail(&bo_va->base.bo_list, &bo->va);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002128
2129 return bo_va;
2130}
2131
Christian König73fb16e2017-08-16 11:13:48 +02002132
2133/**
2134 * amdgpu_vm_bo_insert_mapping - insert a new mapping
2135 *
2136 * @adev: amdgpu_device pointer
2137 * @bo_va: bo_va to store the address
2138 * @mapping: the mapping to insert
2139 *
2140 * Insert a new mapping into all structures.
2141 */
2142static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2143 struct amdgpu_bo_va *bo_va,
2144 struct amdgpu_bo_va_mapping *mapping)
2145{
2146 struct amdgpu_vm *vm = bo_va->base.vm;
2147 struct amdgpu_bo *bo = bo_va->base.bo;
2148
Christian Königaebc5e62017-09-06 16:55:16 +02002149 mapping->bo_va = bo_va;
Christian König73fb16e2017-08-16 11:13:48 +02002150 list_add(&mapping->list, &bo_va->invalids);
2151 amdgpu_vm_it_insert(mapping, &vm->va);
2152
2153 if (mapping->flags & AMDGPU_PTE_PRT)
2154 amdgpu_vm_prt_get(adev);
2155
2156 if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2157 spin_lock(&vm->status_lock);
Christian König481c2e92017-09-01 14:46:19 +02002158 if (list_empty(&bo_va->base.vm_status))
2159 list_add(&bo_va->base.vm_status, &vm->moved);
Christian König73fb16e2017-08-16 11:13:48 +02002160 spin_unlock(&vm->status_lock);
2161 }
2162 trace_amdgpu_vm_bo_map(bo_va, mapping);
2163}
2164
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002165/**
2166 * amdgpu_vm_bo_map - map bo inside a vm
2167 *
2168 * @adev: amdgpu_device pointer
2169 * @bo_va: bo_va to store the address
2170 * @saddr: where to map the BO
2171 * @offset: requested offset in the BO
2172 * @flags: attributes of pages (read/write/valid/etc.)
2173 *
2174 * Add a mapping of the BO at the specefied addr into the VM.
2175 * Returns 0 for success, error for failure.
2176 *
Chunming Zhou49b02b12015-11-13 14:18:38 +08002177 * Object has to be reserved and unreserved outside!
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002178 */
2179int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2180 struct amdgpu_bo_va *bo_va,
2181 uint64_t saddr, uint64_t offset,
Christian König268c3002017-01-18 14:49:43 +01002182 uint64_t size, uint64_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002183{
Christian Königa9f87f62017-03-30 14:03:59 +02002184 struct amdgpu_bo_va_mapping *mapping, *tmp;
Christian Königec681542017-08-01 10:51:43 +02002185 struct amdgpu_bo *bo = bo_va->base.bo;
2186 struct amdgpu_vm *vm = bo_va->base.vm;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002187 uint64_t eaddr;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002188
Christian König0be52de2015-05-18 14:37:27 +02002189 /* validate the parameters */
2190 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
Chunming Zhou49b02b12015-11-13 14:18:38 +08002191 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
Christian König0be52de2015-05-18 14:37:27 +02002192 return -EINVAL;
Christian König0be52de2015-05-18 14:37:27 +02002193
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002194 /* make sure object fit at this offset */
Felix Kuehling005ae952015-11-23 17:43:48 -05002195 eaddr = saddr + size - 1;
Christian Königa5f6b5b2017-01-30 11:01:38 +01002196 if (saddr >= eaddr ||
Christian Königec681542017-08-01 10:51:43 +02002197 (bo && offset + size > amdgpu_bo_size(bo)))
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002198 return -EINVAL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002199
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002200 saddr /= AMDGPU_GPU_PAGE_SIZE;
2201 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2202
Christian Königa9f87f62017-03-30 14:03:59 +02002203 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2204 if (tmp) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002205 /* bo and tmp overlap, invalid addr */
2206 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
Christian Königec681542017-08-01 10:51:43 +02002207 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
Christian Königa9f87f62017-03-30 14:03:59 +02002208 tmp->start, tmp->last + 1);
Christian König663e4572017-03-13 10:13:37 +01002209 return -EINVAL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002210 }
2211
2212 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
Christian König663e4572017-03-13 10:13:37 +01002213 if (!mapping)
2214 return -ENOMEM;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002215
Christian Königa9f87f62017-03-30 14:03:59 +02002216 mapping->start = saddr;
2217 mapping->last = eaddr;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002218 mapping->offset = offset;
2219 mapping->flags = flags;
2220
Christian König73fb16e2017-08-16 11:13:48 +02002221 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
Christian König4388fc22017-03-13 10:13:36 +01002222
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002223 return 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002224}
2225
2226/**
Christian König80f95c52017-03-13 10:13:39 +01002227 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2228 *
2229 * @adev: amdgpu_device pointer
2230 * @bo_va: bo_va to store the address
2231 * @saddr: where to map the BO
2232 * @offset: requested offset in the BO
2233 * @flags: attributes of pages (read/write/valid/etc.)
2234 *
2235 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2236 * mappings as we do so.
2237 * Returns 0 for success, error for failure.
2238 *
2239 * Object has to be reserved and unreserved outside!
2240 */
2241int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2242 struct amdgpu_bo_va *bo_va,
2243 uint64_t saddr, uint64_t offset,
2244 uint64_t size, uint64_t flags)
2245{
2246 struct amdgpu_bo_va_mapping *mapping;
Christian Königec681542017-08-01 10:51:43 +02002247 struct amdgpu_bo *bo = bo_va->base.bo;
Christian König80f95c52017-03-13 10:13:39 +01002248 uint64_t eaddr;
2249 int r;
2250
2251 /* validate the parameters */
2252 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2253 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2254 return -EINVAL;
2255
2256 /* make sure object fit at this offset */
2257 eaddr = saddr + size - 1;
2258 if (saddr >= eaddr ||
Christian Königec681542017-08-01 10:51:43 +02002259 (bo && offset + size > amdgpu_bo_size(bo)))
Christian König80f95c52017-03-13 10:13:39 +01002260 return -EINVAL;
2261
2262 /* Allocate all the needed memory */
2263 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2264 if (!mapping)
2265 return -ENOMEM;
2266
Christian Königec681542017-08-01 10:51:43 +02002267 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
Christian König80f95c52017-03-13 10:13:39 +01002268 if (r) {
2269 kfree(mapping);
2270 return r;
2271 }
2272
2273 saddr /= AMDGPU_GPU_PAGE_SIZE;
2274 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2275
Christian Königa9f87f62017-03-30 14:03:59 +02002276 mapping->start = saddr;
2277 mapping->last = eaddr;
Christian König80f95c52017-03-13 10:13:39 +01002278 mapping->offset = offset;
2279 mapping->flags = flags;
2280
Christian König73fb16e2017-08-16 11:13:48 +02002281 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
Christian König80f95c52017-03-13 10:13:39 +01002282
2283 return 0;
2284}
2285
2286/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002287 * amdgpu_vm_bo_unmap - remove bo mapping from vm
2288 *
2289 * @adev: amdgpu_device pointer
2290 * @bo_va: bo_va to remove the address from
2291 * @saddr: where to the BO is mapped
2292 *
2293 * Remove a mapping of the BO at the specefied addr from the VM.
2294 * Returns 0 for success, error for failure.
2295 *
Chunming Zhou49b02b12015-11-13 14:18:38 +08002296 * Object has to be reserved and unreserved outside!
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002297 */
2298int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2299 struct amdgpu_bo_va *bo_va,
2300 uint64_t saddr)
2301{
2302 struct amdgpu_bo_va_mapping *mapping;
Christian Königec681542017-08-01 10:51:43 +02002303 struct amdgpu_vm *vm = bo_va->base.vm;
Christian König7fc11952015-07-30 11:53:42 +02002304 bool valid = true;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002305
Christian König6c7fc502015-06-05 20:56:17 +02002306 saddr /= AMDGPU_GPU_PAGE_SIZE;
Christian König32b41ac2016-03-08 18:03:27 +01002307
Christian König7fc11952015-07-30 11:53:42 +02002308 list_for_each_entry(mapping, &bo_va->valids, list) {
Christian Königa9f87f62017-03-30 14:03:59 +02002309 if (mapping->start == saddr)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002310 break;
2311 }
2312
Christian König7fc11952015-07-30 11:53:42 +02002313 if (&mapping->list == &bo_va->valids) {
2314 valid = false;
2315
2316 list_for_each_entry(mapping, &bo_va->invalids, list) {
Christian Königa9f87f62017-03-30 14:03:59 +02002317 if (mapping->start == saddr)
Christian König7fc11952015-07-30 11:53:42 +02002318 break;
2319 }
2320
Christian König32b41ac2016-03-08 18:03:27 +01002321 if (&mapping->list == &bo_va->invalids)
Christian König7fc11952015-07-30 11:53:42 +02002322 return -ENOENT;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002323 }
Christian König32b41ac2016-03-08 18:03:27 +01002324
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002325 list_del(&mapping->list);
Christian Königa9f87f62017-03-30 14:03:59 +02002326 amdgpu_vm_it_remove(mapping, &vm->va);
Christian Königaebc5e62017-09-06 16:55:16 +02002327 mapping->bo_va = NULL;
Christian König93e3e432015-06-09 16:58:33 +02002328 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002329
Christian Könige17841b2016-03-08 17:52:01 +01002330 if (valid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002331 list_add(&mapping->list, &vm->freed);
Christian Könige17841b2016-03-08 17:52:01 +01002332 else
Christian König284710f2017-01-30 11:09:31 +01002333 amdgpu_vm_free_mapping(adev, vm, mapping,
2334 bo_va->last_pt_update);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002335
2336 return 0;
2337}
2338
2339/**
Christian Königdc54d3d2017-03-13 10:13:38 +01002340 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2341 *
2342 * @adev: amdgpu_device pointer
2343 * @vm: VM structure to use
2344 * @saddr: start of the range
2345 * @size: size of the range
2346 *
2347 * Remove all mappings in a range, split them as appropriate.
2348 * Returns 0 for success, error for failure.
2349 */
2350int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2351 struct amdgpu_vm *vm,
2352 uint64_t saddr, uint64_t size)
2353{
2354 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
Christian Königdc54d3d2017-03-13 10:13:38 +01002355 LIST_HEAD(removed);
2356 uint64_t eaddr;
2357
2358 eaddr = saddr + size - 1;
2359 saddr /= AMDGPU_GPU_PAGE_SIZE;
2360 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2361
2362 /* Allocate all the needed memory */
2363 before = kzalloc(sizeof(*before), GFP_KERNEL);
2364 if (!before)
2365 return -ENOMEM;
Junwei Zhang27f6d612017-03-16 16:09:24 +08002366 INIT_LIST_HEAD(&before->list);
Christian Königdc54d3d2017-03-13 10:13:38 +01002367
2368 after = kzalloc(sizeof(*after), GFP_KERNEL);
2369 if (!after) {
2370 kfree(before);
2371 return -ENOMEM;
2372 }
Junwei Zhang27f6d612017-03-16 16:09:24 +08002373 INIT_LIST_HEAD(&after->list);
Christian Königdc54d3d2017-03-13 10:13:38 +01002374
2375 /* Now gather all removed mappings */
Christian Königa9f87f62017-03-30 14:03:59 +02002376 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2377 while (tmp) {
Christian Königdc54d3d2017-03-13 10:13:38 +01002378 /* Remember mapping split at the start */
Christian Königa9f87f62017-03-30 14:03:59 +02002379 if (tmp->start < saddr) {
2380 before->start = tmp->start;
2381 before->last = saddr - 1;
Christian Königdc54d3d2017-03-13 10:13:38 +01002382 before->offset = tmp->offset;
2383 before->flags = tmp->flags;
2384 list_add(&before->list, &tmp->list);
2385 }
2386
2387 /* Remember mapping split at the end */
Christian Königa9f87f62017-03-30 14:03:59 +02002388 if (tmp->last > eaddr) {
2389 after->start = eaddr + 1;
2390 after->last = tmp->last;
Christian Königdc54d3d2017-03-13 10:13:38 +01002391 after->offset = tmp->offset;
Christian Königa9f87f62017-03-30 14:03:59 +02002392 after->offset += after->start - tmp->start;
Christian Königdc54d3d2017-03-13 10:13:38 +01002393 after->flags = tmp->flags;
2394 list_add(&after->list, &tmp->list);
2395 }
2396
2397 list_del(&tmp->list);
2398 list_add(&tmp->list, &removed);
Christian Königa9f87f62017-03-30 14:03:59 +02002399
2400 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
Christian Königdc54d3d2017-03-13 10:13:38 +01002401 }
2402
2403 /* And free them up */
2404 list_for_each_entry_safe(tmp, next, &removed, list) {
Christian Königa9f87f62017-03-30 14:03:59 +02002405 amdgpu_vm_it_remove(tmp, &vm->va);
Christian Königdc54d3d2017-03-13 10:13:38 +01002406 list_del(&tmp->list);
2407
Christian Königa9f87f62017-03-30 14:03:59 +02002408 if (tmp->start < saddr)
2409 tmp->start = saddr;
2410 if (tmp->last > eaddr)
2411 tmp->last = eaddr;
Christian Königdc54d3d2017-03-13 10:13:38 +01002412
Christian Königaebc5e62017-09-06 16:55:16 +02002413 tmp->bo_va = NULL;
Christian Königdc54d3d2017-03-13 10:13:38 +01002414 list_add(&tmp->list, &vm->freed);
2415 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2416 }
2417
Junwei Zhang27f6d612017-03-16 16:09:24 +08002418 /* Insert partial mapping before the range */
2419 if (!list_empty(&before->list)) {
Christian Königa9f87f62017-03-30 14:03:59 +02002420 amdgpu_vm_it_insert(before, &vm->va);
Christian Königdc54d3d2017-03-13 10:13:38 +01002421 if (before->flags & AMDGPU_PTE_PRT)
2422 amdgpu_vm_prt_get(adev);
2423 } else {
2424 kfree(before);
2425 }
2426
2427 /* Insert partial mapping after the range */
Junwei Zhang27f6d612017-03-16 16:09:24 +08002428 if (!list_empty(&after->list)) {
Christian Königa9f87f62017-03-30 14:03:59 +02002429 amdgpu_vm_it_insert(after, &vm->va);
Christian Königdc54d3d2017-03-13 10:13:38 +01002430 if (after->flags & AMDGPU_PTE_PRT)
2431 amdgpu_vm_prt_get(adev);
2432 } else {
2433 kfree(after);
2434 }
2435
2436 return 0;
2437}
2438
2439/**
Christian Königaebc5e62017-09-06 16:55:16 +02002440 * amdgpu_vm_bo_lookup_mapping - find mapping by address
2441 *
2442 * @vm: the requested VM
2443 *
2444 * Find a mapping by it's address.
2445 */
2446struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2447 uint64_t addr)
2448{
2449 return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2450}
2451
2452/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002453 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2454 *
2455 * @adev: amdgpu_device pointer
2456 * @bo_va: requested bo_va
2457 *
Christian König8843dbb2016-01-26 12:17:11 +01002458 * Remove @bo_va->bo from the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002459 *
2460 * Object have to be reserved!
2461 */
2462void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2463 struct amdgpu_bo_va *bo_va)
2464{
2465 struct amdgpu_bo_va_mapping *mapping, *next;
Christian Königec681542017-08-01 10:51:43 +02002466 struct amdgpu_vm *vm = bo_va->base.vm;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002467
Christian Königec681542017-08-01 10:51:43 +02002468 list_del(&bo_va->base.bo_list);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002469
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002470 spin_lock(&vm->status_lock);
Christian Königec681542017-08-01 10:51:43 +02002471 list_del(&bo_va->base.vm_status);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002472 spin_unlock(&vm->status_lock);
2473
Christian König7fc11952015-07-30 11:53:42 +02002474 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002475 list_del(&mapping->list);
Christian Königa9f87f62017-03-30 14:03:59 +02002476 amdgpu_vm_it_remove(mapping, &vm->va);
Christian Königaebc5e62017-09-06 16:55:16 +02002477 mapping->bo_va = NULL;
Christian König93e3e432015-06-09 16:58:33 +02002478 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
Christian König7fc11952015-07-30 11:53:42 +02002479 list_add(&mapping->list, &vm->freed);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002480 }
Christian König7fc11952015-07-30 11:53:42 +02002481 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2482 list_del(&mapping->list);
Christian Königa9f87f62017-03-30 14:03:59 +02002483 amdgpu_vm_it_remove(mapping, &vm->va);
Christian König284710f2017-01-30 11:09:31 +01002484 amdgpu_vm_free_mapping(adev, vm, mapping,
2485 bo_va->last_pt_update);
Christian König7fc11952015-07-30 11:53:42 +02002486 }
Christian König32b41ac2016-03-08 18:03:27 +01002487
Chris Wilsonf54d1862016-10-25 13:00:45 +01002488 dma_fence_put(bo_va->last_pt_update);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002489 kfree(bo_va);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002490}
2491
2492/**
2493 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2494 *
2495 * @adev: amdgpu_device pointer
2496 * @vm: requested vm
2497 * @bo: amdgpu buffer object
2498 *
Christian König8843dbb2016-01-26 12:17:11 +01002499 * Mark @bo as invalid.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002500 */
2501void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
Christian König3f3333f2017-08-03 14:02:13 +02002502 struct amdgpu_bo *bo, bool evicted)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002503{
Christian Königec681542017-08-01 10:51:43 +02002504 struct amdgpu_vm_bo_base *bo_base;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002505
Christian Königec681542017-08-01 10:51:43 +02002506 list_for_each_entry(bo_base, &bo->va, bo_list) {
Christian König3f3333f2017-08-03 14:02:13 +02002507 struct amdgpu_vm *vm = bo_base->vm;
2508
Christian König3d7d4d32017-08-23 16:13:33 +02002509 bo_base->moved = true;
Christian König3f3333f2017-08-03 14:02:13 +02002510 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2511 spin_lock(&bo_base->vm->status_lock);
Christian König73fb16e2017-08-16 11:13:48 +02002512 if (bo->tbo.type == ttm_bo_type_kernel)
2513 list_move(&bo_base->vm_status, &vm->evicted);
2514 else
2515 list_move_tail(&bo_base->vm_status,
2516 &vm->evicted);
Christian König3f3333f2017-08-03 14:02:13 +02002517 spin_unlock(&bo_base->vm->status_lock);
2518 continue;
2519 }
2520
Christian Königea097292017-08-09 14:15:46 +02002521 if (bo->tbo.type == ttm_bo_type_kernel) {
2522 spin_lock(&bo_base->vm->status_lock);
2523 if (list_empty(&bo_base->vm_status))
2524 list_add(&bo_base->vm_status, &vm->relocated);
2525 spin_unlock(&bo_base->vm->status_lock);
Christian König3f3333f2017-08-03 14:02:13 +02002526 continue;
Christian Königea097292017-08-09 14:15:46 +02002527 }
Christian König3f3333f2017-08-03 14:02:13 +02002528
Christian Königec681542017-08-01 10:51:43 +02002529 spin_lock(&bo_base->vm->status_lock);
2530 if (list_empty(&bo_base->vm_status))
Christian König481c2e92017-09-01 14:46:19 +02002531 list_add(&bo_base->vm_status, &vm->moved);
Christian Königec681542017-08-01 10:51:43 +02002532 spin_unlock(&bo_base->vm->status_lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002533 }
2534}
2535
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002536static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2537{
2538 /* Total bits covered by PD + PTs */
2539 unsigned bits = ilog2(vm_size) + 18;
2540
2541 /* Make sure the PD is 4K in size up to 8GB address space.
2542 Above that split equal between PD and PTs */
2543 if (vm_size <= 8)
2544 return (bits - 9);
2545 else
2546 return ((bits + 3) / 2);
2547}
2548
2549/**
Roger Hed07f14b2017-08-15 16:05:59 +08002550 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002551 *
2552 * @adev: amdgpu_device pointer
2553 * @vm_size: the default vm size if it's set auto
2554 */
Christian Königfdd5faa2017-11-04 16:51:44 +01002555void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
Christian Königf3368122017-11-23 12:57:18 +01002556 uint32_t fragment_size_default, unsigned max_level,
2557 unsigned max_bits)
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002558{
Christian König36539dc2017-11-23 11:16:05 +01002559 uint64_t tmp;
2560
2561 /* adjust vm size first */
Christian Königf3368122017-11-23 12:57:18 +01002562 if (amdgpu_vm_size != -1) {
2563 unsigned max_size = 1 << (max_bits - 30);
2564
Christian Königfdd5faa2017-11-04 16:51:44 +01002565 vm_size = amdgpu_vm_size;
Christian Königf3368122017-11-23 12:57:18 +01002566 if (vm_size > max_size) {
2567 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2568 amdgpu_vm_size, max_size);
2569 vm_size = max_size;
2570 }
2571 }
Christian Königfdd5faa2017-11-04 16:51:44 +01002572
2573 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
Christian König36539dc2017-11-23 11:16:05 +01002574
2575 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
Christian König97489122017-11-27 16:22:05 +01002576 if (amdgpu_vm_block_size != -1)
2577 tmp >>= amdgpu_vm_block_size - 9;
Christian König36539dc2017-11-23 11:16:05 +01002578 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2579 adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002580
Christian Königb38f41e2017-11-22 17:00:35 +01002581 /* block size depends on vm size and hw setup*/
Christian König97489122017-11-27 16:22:05 +01002582 if (amdgpu_vm_block_size != -1)
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002583 adev->vm_manager.block_size =
Christian König97489122017-11-27 16:22:05 +01002584 min((unsigned)amdgpu_vm_block_size, max_bits
2585 - AMDGPU_GPU_PAGE_SHIFT
2586 - 9 * adev->vm_manager.num_level);
2587 else if (adev->vm_manager.num_level > 1)
2588 adev->vm_manager.block_size = 9;
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002589 else
Christian König97489122017-11-27 16:22:05 +01002590 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002591
Christian Königb38f41e2017-11-22 17:00:35 +01002592 if (amdgpu_vm_fragment_size == -1)
2593 adev->vm_manager.fragment_size = fragment_size_default;
2594 else
2595 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
Roger Hed07f14b2017-08-15 16:05:59 +08002596
Christian König36539dc2017-11-23 11:16:05 +01002597 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2598 vm_size, adev->vm_manager.num_level + 1,
2599 adev->vm_manager.block_size,
Christian Königfdd5faa2017-11-04 16:51:44 +01002600 adev->vm_manager.fragment_size);
Junwei Zhangbab4fee2017-04-05 13:54:56 +08002601}
2602
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002603/**
2604 * amdgpu_vm_init - initialize a vm instance
2605 *
2606 * @adev: amdgpu_device pointer
2607 * @vm: requested vm
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -04002608 * @vm_context: Indicates if it GFX or Compute context
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002609 *
Christian König8843dbb2016-01-26 12:17:11 +01002610 * Init @vm fields.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002611 */
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -04002612int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
Felix Kuehling02208442017-08-25 20:40:26 -04002613 int vm_context, unsigned int pasid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002614{
2615 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
Zhang, Jerry36b32a62017-03-29 16:08:32 +08002616 AMDGPU_VM_PTE_COUNT(adev) * 8);
Christian König2d55e452016-02-08 17:37:38 +01002617 unsigned ring_instance;
2618 struct amdgpu_ring *ring;
Lucas Stach1b1f42d2017-12-06 17:49:39 +01002619 struct drm_sched_rq *rq;
Chunming Zhou36bbf3b2017-04-20 16:17:34 +08002620 int r, i;
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04002621 u64 flags;
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002622 uint64_t init_pde_value = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002623
Davidlohr Buesof808c132017-09-08 16:15:08 -07002624 vm->va = RB_ROOT_CACHED;
Chunming Zhou031e2982016-04-25 10:19:13 +08002625 vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
Chunming Zhou36bbf3b2017-04-20 16:17:34 +08002626 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2627 vm->reserved_vmid[i] = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002628 spin_lock_init(&vm->status_lock);
Christian König3f3333f2017-08-03 14:02:13 +02002629 INIT_LIST_HEAD(&vm->evicted);
Christian Königea097292017-08-09 14:15:46 +02002630 INIT_LIST_HEAD(&vm->relocated);
Christian König27c7b9a2017-08-01 11:27:36 +02002631 INIT_LIST_HEAD(&vm->moved);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002632 INIT_LIST_HEAD(&vm->freed);
Christian König20250212016-03-08 17:58:35 +01002633
Christian König2bd9ccf2016-02-01 12:53:58 +01002634 /* create scheduler entity for page table updates */
Christian König2d55e452016-02-08 17:37:38 +01002635
2636 ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
2637 ring_instance %= adev->vm_manager.vm_pte_num_rings;
2638 ring = adev->vm_manager.vm_pte_rings[ring_instance];
Lucas Stach1b1f42d2017-12-06 17:49:39 +01002639 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
2640 r = drm_sched_entity_init(&ring->sched, &vm->entity,
Monk Liub3eebe32017-10-23 12:23:29 +08002641 rq, amdgpu_sched_jobs, NULL);
Christian König2bd9ccf2016-02-01 12:53:58 +01002642 if (r)
Christian Königf566ceb2016-10-27 20:04:38 +02002643 return r;
Christian König2bd9ccf2016-02-01 12:53:58 +01002644
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002645 vm->pte_support_ats = false;
2646
2647 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -04002648 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2649 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002650
2651 if (adev->asic_type == CHIP_RAVEN) {
2652 vm->pte_support_ats = true;
Yong Zhao6d16dac2017-08-31 15:55:00 -04002653 init_pde_value = AMDGPU_PTE_DEFAULT_ATC
2654 | AMDGPU_PDE_PTE;
2655
Yong Zhao51ac7ee2017-07-27 12:48:22 -04002656 }
2657 } else
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -04002658 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2659 AMDGPU_VM_USE_CPU_FOR_GFX);
2660 DRM_DEBUG_DRIVER("VM update mode is %s\n",
2661 vm->use_cpu_for_update ? "CPU" : "SDMA");
2662 WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
2663 "CPU update of VM recommended only for large BAR system\n");
Christian Königd5884512017-09-08 14:09:41 +02002664 vm->last_update = NULL;
Bas Nieuwenhuizen05906de2015-08-14 20:08:40 +02002665
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04002666 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
2667 AMDGPU_GEM_CREATE_VRAM_CLEARED;
2668 if (vm->use_cpu_for_update)
2669 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
2670 else
2671 flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
2672 AMDGPU_GEM_CREATE_SHADOW);
2673
Christian Königf566ceb2016-10-27 20:04:38 +02002674 r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
Alex Deucher857d9132015-08-27 00:14:16 -04002675 AMDGPU_GEM_DOMAIN_VRAM,
Harish Kasiviswanathan3c824172017-05-11 15:50:08 -04002676 flags,
Christian König3f3333f2017-08-03 14:02:13 +02002677 NULL, NULL, init_pde_value, &vm->root.base.bo);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002678 if (r)
Christian König2bd9ccf2016-02-01 12:53:58 +01002679 goto error_free_sched_entity;
2680
Christian König3f3333f2017-08-03 14:02:13 +02002681 vm->root.base.vm = vm;
2682 list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
2683 INIT_LIST_HEAD(&vm->root.base.vm_status);
Christian König0a096fb2017-07-12 10:01:48 +02002684
2685 if (vm->use_cpu_for_update) {
Christian König3f3333f2017-08-03 14:02:13 +02002686 r = amdgpu_bo_reserve(vm->root.base.bo, false);
Christian König0a096fb2017-07-12 10:01:48 +02002687 if (r)
2688 goto error_free_root;
Christian König0a096fb2017-07-12 10:01:48 +02002689
Christian König3f3333f2017-08-03 14:02:13 +02002690 r = amdgpu_bo_kmap(vm->root.base.bo, NULL);
Felix Kuehlingca290da2017-08-25 20:15:04 -04002691 amdgpu_bo_unreserve(vm->root.base.bo);
Christian König2bd9ccf2016-02-01 12:53:58 +01002692 if (r)
2693 goto error_free_root;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002694 }
2695
Felix Kuehling02208442017-08-25 20:40:26 -04002696 if (pasid) {
2697 unsigned long flags;
2698
2699 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2700 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2701 GFP_ATOMIC);
2702 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2703 if (r < 0)
2704 goto error_free_root;
2705
2706 vm->pasid = pasid;
2707 }
2708
Felix Kuehlinga2f14822017-08-26 02:43:06 -04002709 INIT_KFIFO(vm->faults);
Felix Kuehlingc98171c2017-09-21 16:26:41 -04002710 vm->fault_credit = 16;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002711
2712 return 0;
2713
2714error_free_root:
Christian König3f3333f2017-08-03 14:02:13 +02002715 amdgpu_bo_unref(&vm->root.base.bo->shadow);
2716 amdgpu_bo_unref(&vm->root.base.bo);
2717 vm->root.base.bo = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002718
2719error_free_sched_entity:
Lucas Stach1b1f42d2017-12-06 17:49:39 +01002720 drm_sched_entity_fini(&ring->sched, &vm->entity);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002721
2722 return r;
2723}
2724
2725/**
Christian Königf566ceb2016-10-27 20:04:38 +02002726 * amdgpu_vm_free_levels - free PD/PT levels
2727 *
2728 * @level: PD/PT starting level to free
2729 *
2730 * Free the page directory or page table level and all sub levels.
2731 */
2732static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
2733{
2734 unsigned i;
2735
Christian König3f3333f2017-08-03 14:02:13 +02002736 if (level->base.bo) {
2737 list_del(&level->base.bo_list);
2738 list_del(&level->base.vm_status);
2739 amdgpu_bo_unref(&level->base.bo->shadow);
2740 amdgpu_bo_unref(&level->base.bo);
Christian Königf566ceb2016-10-27 20:04:38 +02002741 }
2742
2743 if (level->entries)
2744 for (i = 0; i <= level->last_entry_used; i++)
2745 amdgpu_vm_free_levels(&level->entries[i]);
2746
Michal Hocko20981052017-05-17 14:23:12 +02002747 kvfree(level->entries);
Christian Königf566ceb2016-10-27 20:04:38 +02002748}
2749
2750/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002751 * amdgpu_vm_fini - tear down a vm instance
2752 *
2753 * @adev: amdgpu_device pointer
2754 * @vm: requested vm
2755 *
Christian König8843dbb2016-01-26 12:17:11 +01002756 * Tear down @vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002757 * Unbind the VM and remove all bos from the vm bo list
2758 */
2759void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2760{
2761 struct amdgpu_bo_va_mapping *mapping, *tmp;
Christian König4388fc22017-03-13 10:13:36 +01002762 bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
Christian König2642cf12017-10-13 17:24:31 +02002763 struct amdgpu_bo *root;
Felix Kuehlinga2f14822017-08-26 02:43:06 -04002764 u64 fault;
Christian König2642cf12017-10-13 17:24:31 +02002765 int i, r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002766
Felix Kuehlinga2f14822017-08-26 02:43:06 -04002767 /* Clear pending page faults from IH when the VM is destroyed */
2768 while (kfifo_get(&vm->faults, &fault))
2769 amdgpu_ih_clear_fault(adev, fault);
2770
Felix Kuehling02208442017-08-25 20:40:26 -04002771 if (vm->pasid) {
2772 unsigned long flags;
2773
2774 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2775 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2776 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2777 }
2778
Lucas Stach1b1f42d2017-12-06 17:49:39 +01002779 drm_sched_entity_fini(vm->entity.sched, &vm->entity);
Christian König2bd9ccf2016-02-01 12:53:58 +01002780
Davidlohr Buesof808c132017-09-08 16:15:08 -07002781 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002782 dev_err(adev->dev, "still active bo inside vm\n");
2783 }
Davidlohr Buesof808c132017-09-08 16:15:08 -07002784 rbtree_postorder_for_each_entry_safe(mapping, tmp,
2785 &vm->va.rb_root, rb) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002786 list_del(&mapping->list);
Christian Königa9f87f62017-03-30 14:03:59 +02002787 amdgpu_vm_it_remove(mapping, &vm->va);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002788 kfree(mapping);
2789 }
2790 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
Christian König4388fc22017-03-13 10:13:36 +01002791 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
Christian König451bc8e2017-02-14 16:02:52 +01002792 amdgpu_vm_prt_fini(adev, vm);
Christian König4388fc22017-03-13 10:13:36 +01002793 prt_fini_needed = false;
Christian König451bc8e2017-02-14 16:02:52 +01002794 }
Christian König284710f2017-01-30 11:09:31 +01002795
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002796 list_del(&mapping->list);
Christian König451bc8e2017-02-14 16:02:52 +01002797 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002798 }
2799
Christian König2642cf12017-10-13 17:24:31 +02002800 root = amdgpu_bo_ref(vm->root.base.bo);
2801 r = amdgpu_bo_reserve(root, true);
2802 if (r) {
2803 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2804 } else {
2805 amdgpu_vm_free_levels(&vm->root);
2806 amdgpu_bo_unreserve(root);
2807 }
2808 amdgpu_bo_unref(&root);
Christian Königd5884512017-09-08 14:09:41 +02002809 dma_fence_put(vm->last_update);
Chunming Zhou1e9ef262017-04-20 16:18:48 +08002810 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2811 amdgpu_vm_free_reserved_vmid(adev, vm, i);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04002812}
Christian Königea89f8c2015-11-15 20:52:06 +01002813
2814/**
Felix Kuehlingc98171c2017-09-21 16:26:41 -04002815 * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
2816 *
2817 * @adev: amdgpu_device pointer
2818 * @pasid: PASID do identify the VM
2819 *
2820 * This function is expected to be called in interrupt context. Returns
2821 * true if there was fault credit, false otherwise
2822 */
2823bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
2824 unsigned int pasid)
2825{
2826 struct amdgpu_vm *vm;
2827
2828 spin_lock(&adev->vm_manager.pasid_lock);
2829 vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
2830 spin_unlock(&adev->vm_manager.pasid_lock);
2831 if (!vm)
2832 /* VM not found, can't track fault credit */
2833 return true;
2834
2835 /* No lock needed. only accessed by IRQ handler */
2836 if (!vm->fault_credit)
2837 /* Too many faults in this VM */
2838 return false;
2839
2840 vm->fault_credit--;
2841 return true;
2842}
2843
2844/**
Christian Königa9a78b32016-01-21 10:19:11 +01002845 * amdgpu_vm_manager_init - init the VM manager
2846 *
2847 * @adev: amdgpu_device pointer
2848 *
2849 * Initialize the VM manager structures
2850 */
2851void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2852{
Christian König76456702017-04-06 17:52:39 +02002853 unsigned i, j;
Christian Königa9a78b32016-01-21 10:19:11 +01002854
Christian König76456702017-04-06 17:52:39 +02002855 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2856 struct amdgpu_vm_id_manager *id_mgr =
2857 &adev->vm_manager.id_mgr[i];
Christian Königa9a78b32016-01-21 10:19:11 +01002858
Christian König76456702017-04-06 17:52:39 +02002859 mutex_init(&id_mgr->lock);
2860 INIT_LIST_HEAD(&id_mgr->ids_lru);
Chunming Zhouc3505772017-04-21 15:51:04 +08002861 atomic_set(&id_mgr->reserved_vmid_num, 0);
Christian König76456702017-04-06 17:52:39 +02002862
2863 /* skip over VMID 0, since it is the system VM */
2864 for (j = 1; j < id_mgr->num_ids; ++j) {
2865 amdgpu_vm_reset_id(adev, i, j);
2866 amdgpu_sync_create(&id_mgr->ids[i].active);
2867 list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
2868 }
Christian König971fe9a92016-03-01 15:09:25 +01002869 }
Christian König2d55e452016-02-08 17:37:38 +01002870
Chris Wilsonf54d1862016-10-25 13:00:45 +01002871 adev->vm_manager.fence_context =
2872 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
Christian König1fbb2e92016-06-01 10:47:36 +02002873 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2874 adev->vm_manager.seqno[i] = 0;
2875
Christian König2d55e452016-02-08 17:37:38 +01002876 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
Christian Königb1c8a812016-05-04 10:34:03 +02002877 atomic64_set(&adev->vm_manager.client_counter, 0);
Christian König284710f2017-01-30 11:09:31 +01002878 spin_lock_init(&adev->vm_manager.prt_lock);
Christian König451bc8e2017-02-14 16:02:52 +01002879 atomic_set(&adev->vm_manager.num_prt_users, 0);
Harish Kasiviswanathan9a4b7d42017-06-09 11:26:57 -04002880
2881 /* If not overridden by the user, by default, only in large BAR systems
2882 * Compute VM tables will be updated by CPU
2883 */
2884#ifdef CONFIG_X86_64
2885 if (amdgpu_vm_update_mode == -1) {
2886 if (amdgpu_vm_is_large_bar(adev))
2887 adev->vm_manager.vm_update_mode =
2888 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2889 else
2890 adev->vm_manager.vm_update_mode = 0;
2891 } else
2892 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2893#else
2894 adev->vm_manager.vm_update_mode = 0;
2895#endif
2896
Felix Kuehling02208442017-08-25 20:40:26 -04002897 idr_init(&adev->vm_manager.pasid_idr);
2898 spin_lock_init(&adev->vm_manager.pasid_lock);
Christian Königa9a78b32016-01-21 10:19:11 +01002899}
2900
2901/**
Christian Königea89f8c2015-11-15 20:52:06 +01002902 * amdgpu_vm_manager_fini - cleanup VM manager
2903 *
2904 * @adev: amdgpu_device pointer
2905 *
2906 * Cleanup the VM manager and free resources.
2907 */
2908void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2909{
Christian König76456702017-04-06 17:52:39 +02002910 unsigned i, j;
Christian Königea89f8c2015-11-15 20:52:06 +01002911
Felix Kuehling02208442017-08-25 20:40:26 -04002912 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
2913 idr_destroy(&adev->vm_manager.pasid_idr);
2914
Christian König76456702017-04-06 17:52:39 +02002915 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2916 struct amdgpu_vm_id_manager *id_mgr =
2917 &adev->vm_manager.id_mgr[i];
Christian Königbcb1ba32016-03-08 15:40:11 +01002918
Christian König76456702017-04-06 17:52:39 +02002919 mutex_destroy(&id_mgr->lock);
2920 for (j = 0; j < AMDGPU_NUM_VM; ++j) {
2921 struct amdgpu_vm_id *id = &id_mgr->ids[j];
2922
2923 amdgpu_sync_free(&id->active);
2924 dma_fence_put(id->flushed_updates);
2925 dma_fence_put(id->last_flush);
2926 }
Christian Königbcb1ba32016-03-08 15:40:11 +01002927 }
Christian Königea89f8c2015-11-15 20:52:06 +01002928}
Chunming Zhoucfbcacf2017-04-24 11:09:04 +08002929
2930int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2931{
2932 union drm_amdgpu_vm *args = data;
Chunming Zhou1e9ef262017-04-20 16:18:48 +08002933 struct amdgpu_device *adev = dev->dev_private;
2934 struct amdgpu_fpriv *fpriv = filp->driver_priv;
2935 int r;
Chunming Zhoucfbcacf2017-04-24 11:09:04 +08002936
2937 switch (args->in.op) {
2938 case AMDGPU_VM_OP_RESERVE_VMID:
Chunming Zhou1e9ef262017-04-20 16:18:48 +08002939 /* current, we only have requirement to reserve vmid from gfxhub */
2940 r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
2941 AMDGPU_GFXHUB);
2942 if (r)
2943 return r;
2944 break;
Chunming Zhoucfbcacf2017-04-24 11:09:04 +08002945 case AMDGPU_VM_OP_UNRESERVE_VMID:
Chunming Zhou1e9ef262017-04-20 16:18:48 +08002946 amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
Chunming Zhoucfbcacf2017-04-24 11:09:04 +08002947 break;
2948 default:
2949 return -EINVAL;
2950 }
2951
2952 return 0;
2953}