blob: 18daa2d64d57340a5a0dfb4bd457a2764f652469 [file] [log] [blame]
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
Christian König1fbb2e92016-06-01 10:47:36 +020028#include <linux/fence-array.h>
Alex Deucherd38ceaf2015-04-20 16:55:21 -040029#include <drm/drmP.h>
30#include <drm/amdgpu_drm.h>
31#include "amdgpu.h"
32#include "amdgpu_trace.h"
33
34/*
35 * GPUVM
36 * GPUVM is similar to the legacy gart on older asics, however
37 * rather than there being a single global gart table
38 * for the entire GPU, there are multiple VM page tables active
39 * at any given time. The VM page tables can contain a mix
40 * vram pages and system memory pages and system memory pages
41 * can be mapped as snooped (cached system pages) or unsnooped
42 * (uncached system pages).
43 * Each VM has an ID associated with it and there is a page table
44 * associated with each VMID. When execting a command buffer,
45 * the kernel tells the the ring what VMID to use for that command
46 * buffer. VMIDs are allocated dynamically as commands are submitted.
47 * The userspace drivers maintain their own address space and the kernel
48 * sets up their pages tables accordingly when they submit their
49 * command buffers and a VMID is assigned.
50 * Cayman/Trinity support up to 8 active VMs at any given time;
51 * SI supports 16.
52 */
53
Christian König4ff37a82016-02-26 16:18:26 +010054/* Special value that no flush is necessary */
55#define AMDGPU_VM_NO_FLUSH (~0ll)
56
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -040057/* Local structure. Encapsulate some VM table update parameters to reduce
58 * the number of function parameters
59 */
60struct amdgpu_vm_update_params {
61 /* address where to copy page table entries from */
62 uint64_t src;
63 /* DMA addresses to use for mapping */
64 dma_addr_t *pages_addr;
65 /* indirect buffer to fill with commands */
66 struct amdgpu_ib *ib;
67};
68
Alex Deucherd38ceaf2015-04-20 16:55:21 -040069/**
70 * amdgpu_vm_num_pde - return the number of page directory entries
71 *
72 * @adev: amdgpu_device pointer
73 *
Christian König8843dbb2016-01-26 12:17:11 +010074 * Calculate the number of page directory entries.
Alex Deucherd38ceaf2015-04-20 16:55:21 -040075 */
76static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
77{
78 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
79}
80
81/**
82 * amdgpu_vm_directory_size - returns the size of the page directory in bytes
83 *
84 * @adev: amdgpu_device pointer
85 *
Christian König8843dbb2016-01-26 12:17:11 +010086 * Calculate the size of the page directory in bytes.
Alex Deucherd38ceaf2015-04-20 16:55:21 -040087 */
88static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
89{
90 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
91}
92
93/**
Christian König56467eb2015-12-11 15:16:32 +010094 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
Alex Deucherd38ceaf2015-04-20 16:55:21 -040095 *
96 * @vm: vm providing the BOs
Christian König3c0eea62015-12-11 14:39:05 +010097 * @validated: head of validation list
Christian König56467eb2015-12-11 15:16:32 +010098 * @entry: entry to add
Alex Deucherd38ceaf2015-04-20 16:55:21 -040099 *
100 * Add the page directory to the list of BOs to
Christian König56467eb2015-12-11 15:16:32 +0100101 * validate for command submission.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400102 */
Christian König56467eb2015-12-11 15:16:32 +0100103void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
104 struct list_head *validated,
105 struct amdgpu_bo_list_entry *entry)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400106{
Christian König56467eb2015-12-11 15:16:32 +0100107 entry->robj = vm->page_directory;
Christian König56467eb2015-12-11 15:16:32 +0100108 entry->priority = 0;
109 entry->tv.bo = &vm->page_directory->tbo;
110 entry->tv.shared = true;
Christian König2f568db2016-02-23 12:36:59 +0100111 entry->user_pages = NULL;
Christian König56467eb2015-12-11 15:16:32 +0100112 list_add(&entry->tv.head, validated);
113}
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400114
Christian König56467eb2015-12-11 15:16:32 +0100115/**
Christian Königee1782c2015-12-11 21:01:23 +0100116 * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
Christian König56467eb2015-12-11 15:16:32 +0100117 *
118 * @vm: vm providing the BOs
Christian König3c0eea62015-12-11 14:39:05 +0100119 * @duplicates: head of duplicates list
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400120 *
Christian Königee1782c2015-12-11 21:01:23 +0100121 * Add the page directory to the BO duplicates list
122 * for command submission.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400123 */
Christian Königee1782c2015-12-11 21:01:23 +0100124void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400125{
Christian Königee1782c2015-12-11 21:01:23 +0100126 unsigned i;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400127
128 /* add the vm page table to the list */
Christian Königee1782c2015-12-11 21:01:23 +0100129 for (i = 0; i <= vm->max_pde_used; ++i) {
130 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400131
Christian Königee1782c2015-12-11 21:01:23 +0100132 if (!entry->robj)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400133 continue;
134
Christian Königee1782c2015-12-11 21:01:23 +0100135 list_add(&entry->tv.head, duplicates);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400136 }
Christian Königeceb8a12016-01-11 15:35:21 +0100137
138}
139
140/**
141 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
142 *
143 * @adev: amdgpu device instance
144 * @vm: vm providing the BOs
145 *
146 * Move the PT BOs to the tail of the LRU.
147 */
148void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
149 struct amdgpu_vm *vm)
150{
151 struct ttm_bo_global *glob = adev->mman.bdev.glob;
152 unsigned i;
153
154 spin_lock(&glob->lru_lock);
155 for (i = 0; i <= vm->max_pde_used; ++i) {
156 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
157
158 if (!entry->robj)
159 continue;
160
161 ttm_bo_move_to_lru_tail(&entry->robj->tbo);
162 }
163 spin_unlock(&glob->lru_lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400164}
165
166/**
167 * amdgpu_vm_grab_id - allocate the next free VMID
168 *
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400169 * @vm: vm to allocate id for
Christian König7f8a5292015-07-20 16:09:40 +0200170 * @ring: ring we want to submit job to
171 * @sync: sync object where we add dependencies
Christian König94dd0a42016-01-18 17:01:42 +0100172 * @fence: fence protecting ID from reuse
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400173 *
Christian König7f8a5292015-07-20 16:09:40 +0200174 * Allocate an id for the vm, adding fences to the sync obj as necessary.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400175 */
Christian König7f8a5292015-07-20 16:09:40 +0200176int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
Christian König4ff37a82016-02-26 16:18:26 +0100177 struct amdgpu_sync *sync, struct fence *fence,
178 unsigned *vm_id, uint64_t *vm_pd_addr)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400179{
Christian König4ff37a82016-02-26 16:18:26 +0100180 uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400181 struct amdgpu_device *adev = ring->adev;
Christian König4ff37a82016-02-26 16:18:26 +0100182 struct fence *updates = sync->last_vm_update;
Christian König8d76001e2016-05-23 16:00:32 +0200183 struct amdgpu_vm_id *id, *idle;
Christian König1fbb2e92016-06-01 10:47:36 +0200184 struct fence **fences;
185 unsigned i;
186 int r = 0;
187
188 fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
189 GFP_KERNEL);
190 if (!fences)
191 return -ENOMEM;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400192
Christian König94dd0a42016-01-18 17:01:42 +0100193 mutex_lock(&adev->vm_manager.lock);
194
Christian König36fd7c52016-05-23 15:30:08 +0200195 /* Check if we have an idle VMID */
Christian König1fbb2e92016-06-01 10:47:36 +0200196 i = 0;
Christian König8d76001e2016-05-23 16:00:32 +0200197 list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
Christian König1fbb2e92016-06-01 10:47:36 +0200198 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
199 if (!fences[i])
Christian König36fd7c52016-05-23 15:30:08 +0200200 break;
Christian König1fbb2e92016-06-01 10:47:36 +0200201 ++i;
Christian König36fd7c52016-05-23 15:30:08 +0200202 }
Christian Königbcb1ba32016-03-08 15:40:11 +0100203
Christian König1fbb2e92016-06-01 10:47:36 +0200204 /* If we can't find a idle VMID to use, wait till one becomes available */
Christian König8d76001e2016-05-23 16:00:32 +0200205 if (&idle->list == &adev->vm_manager.ids_lru) {
Christian König1fbb2e92016-06-01 10:47:36 +0200206 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
207 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
208 struct fence_array *array;
209 unsigned j;
Christian König8d76001e2016-05-23 16:00:32 +0200210
Christian König1fbb2e92016-06-01 10:47:36 +0200211 for (j = 0; j < i; ++j)
212 fence_get(fences[j]);
Christian König8d76001e2016-05-23 16:00:32 +0200213
Christian König1fbb2e92016-06-01 10:47:36 +0200214 array = fence_array_create(i, fences, fence_context,
215 seqno, true);
216 if (!array) {
217 for (j = 0; j < i; ++j)
218 fence_put(fences[j]);
219 kfree(fences);
220 r = -ENOMEM;
221 goto error;
222 }
Christian König8d76001e2016-05-23 16:00:32 +0200223
Christian König8d76001e2016-05-23 16:00:32 +0200224
Christian König1fbb2e92016-06-01 10:47:36 +0200225 r = amdgpu_sync_fence(ring->adev, sync, &array->base);
226 fence_put(&array->base);
227 if (r)
228 goto error;
Christian König8d76001e2016-05-23 16:00:32 +0200229
Christian König1fbb2e92016-06-01 10:47:36 +0200230 mutex_unlock(&adev->vm_manager.lock);
231 return 0;
Christian König8d76001e2016-05-23 16:00:32 +0200232
Christian König1fbb2e92016-06-01 10:47:36 +0200233 }
234 kfree(fences);
Christian König8d76001e2016-05-23 16:00:32 +0200235
Christian König1fbb2e92016-06-01 10:47:36 +0200236 /* Check if we can use a VMID already assigned to this VM */
237 i = ring->idx;
238 do {
239 struct fence *flushed;
Christian König3dab83b2016-06-01 13:31:17 +0200240 bool same_ring = ring->idx == i;
Christian König8d76001e2016-05-23 16:00:32 +0200241
Christian König1fbb2e92016-06-01 10:47:36 +0200242 id = vm->ids[i++];
243 if (i == AMDGPU_MAX_RINGS)
244 i = 0;
245
246 /* Check all the prerequisites to using this VMID */
247 if (!id)
248 continue;
249
250 if (atomic64_read(&id->owner) != vm->client_id)
251 continue;
252
253 if (pd_addr != id->pd_gpu_addr)
254 continue;
255
Christian König3dab83b2016-06-01 13:31:17 +0200256 if (!same_ring &&
Christian König1fbb2e92016-06-01 10:47:36 +0200257 (!id->last_flush || !fence_is_signaled(id->last_flush)))
258 continue;
259
260 flushed = id->flushed_updates;
261 if (updates &&
262 (!flushed || fence_is_later(updates, flushed)))
263 continue;
264
Christian König3dab83b2016-06-01 13:31:17 +0200265 /* Good we can use this VMID. Remember this submission as
266 * user of the VMID.
267 */
Christian König1fbb2e92016-06-01 10:47:36 +0200268 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
269 if (r)
270 goto error;
Christian König8d76001e2016-05-23 16:00:32 +0200271
Christian König1fbb2e92016-06-01 10:47:36 +0200272 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
273 vm->ids[ring->idx] = id;
Christian König8d76001e2016-05-23 16:00:32 +0200274
Christian König1fbb2e92016-06-01 10:47:36 +0200275 *vm_id = id - adev->vm_manager.ids;
276 *vm_pd_addr = AMDGPU_VM_NO_FLUSH;
277 trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
Christian König8d76001e2016-05-23 16:00:32 +0200278
Christian König1fbb2e92016-06-01 10:47:36 +0200279 mutex_unlock(&adev->vm_manager.lock);
280 return 0;
Christian König8d76001e2016-05-23 16:00:32 +0200281
Christian König1fbb2e92016-06-01 10:47:36 +0200282 } while (i != ring->idx);
Chunming Zhou8e9fbeb2016-03-17 11:41:37 +0800283
Christian König1fbb2e92016-06-01 10:47:36 +0200284 /* Still no ID to use? Then use the idle one found earlier */
285 id = idle;
286
287 /* Remember this submission as user of the VMID */
288 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
Christian König832a9022016-02-15 12:33:02 +0100289 if (r)
290 goto error;
Christian König4ff37a82016-02-26 16:18:26 +0100291
Christian König832a9022016-02-15 12:33:02 +0100292 fence_put(id->first);
293 id->first = fence_get(fence);
Christian König4ff37a82016-02-26 16:18:26 +0100294
Christian König41d9eb22016-03-01 16:46:18 +0100295 fence_put(id->last_flush);
296 id->last_flush = NULL;
297
Christian König832a9022016-02-15 12:33:02 +0100298 fence_put(id->flushed_updates);
299 id->flushed_updates = fence_get(updates);
Christian König4ff37a82016-02-26 16:18:26 +0100300
Christian König832a9022016-02-15 12:33:02 +0100301 id->pd_gpu_addr = pd_addr;
Christian König4ff37a82016-02-26 16:18:26 +0100302
Christian König832a9022016-02-15 12:33:02 +0100303 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
Christian König0ea54b92016-05-04 10:20:01 +0200304 atomic64_set(&id->owner, vm->client_id);
Christian König832a9022016-02-15 12:33:02 +0100305 vm->ids[ring->idx] = id;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400306
Christian König832a9022016-02-15 12:33:02 +0100307 *vm_id = id - adev->vm_manager.ids;
308 *vm_pd_addr = pd_addr;
309 trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
310
311error:
Christian König94dd0a42016-01-18 17:01:42 +0100312 mutex_unlock(&adev->vm_manager.lock);
Christian Königa9a78b32016-01-21 10:19:11 +0100313 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400314}
315
316/**
317 * amdgpu_vm_flush - hardware flush the vm
318 *
319 * @ring: ring to use for flush
Christian Königcffadc82016-03-01 13:34:49 +0100320 * @vm_id: vmid number to use
Christian König4ff37a82016-02-26 16:18:26 +0100321 * @pd_addr: address of the page directory
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400322 *
Christian König4ff37a82016-02-26 16:18:26 +0100323 * Emit a VM flush when it is necessary.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400324 */
Christian König41d9eb22016-03-01 16:46:18 +0100325int amdgpu_vm_flush(struct amdgpu_ring *ring,
326 unsigned vm_id, uint64_t pd_addr,
327 uint32_t gds_base, uint32_t gds_size,
328 uint32_t gws_base, uint32_t gws_size,
329 uint32_t oa_base, uint32_t oa_size)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400330{
Christian König971fe9a92016-03-01 15:09:25 +0100331 struct amdgpu_device *adev = ring->adev;
Christian Königbcb1ba32016-03-08 15:40:11 +0100332 struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
Christian Königd564a062016-03-01 15:51:53 +0100333 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
Christian Königbcb1ba32016-03-08 15:40:11 +0100334 id->gds_base != gds_base ||
335 id->gds_size != gds_size ||
336 id->gws_base != gws_base ||
337 id->gws_size != gws_size ||
338 id->oa_base != oa_base ||
339 id->oa_size != oa_size);
Christian König41d9eb22016-03-01 16:46:18 +0100340 int r;
Christian Königd564a062016-03-01 15:51:53 +0100341
342 if (ring->funcs->emit_pipeline_sync && (
Chunming Zhoufe707662016-04-27 18:07:41 +0800343 pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed ||
344 ring->type == AMDGPU_RING_TYPE_COMPUTE))
Christian Königd564a062016-03-01 15:51:53 +0100345 amdgpu_ring_emit_pipeline_sync(ring);
Christian König971fe9a92016-03-01 15:09:25 +0100346
Monk Liuc5637832016-04-19 20:11:32 +0800347 if (ring->funcs->emit_vm_flush &&
348 pd_addr != AMDGPU_VM_NO_FLUSH) {
Christian König41d9eb22016-03-01 16:46:18 +0100349 struct fence *fence;
350
Christian Königcffadc82016-03-01 13:34:49 +0100351 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id);
352 amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr);
Christian König41d9eb22016-03-01 16:46:18 +0100353
Christian König3dab83b2016-06-01 13:31:17 +0200354 r = amdgpu_fence_emit(ring, &fence);
355 if (r)
356 return r;
357
Christian König41d9eb22016-03-01 16:46:18 +0100358 mutex_lock(&adev->vm_manager.lock);
Christian König3dab83b2016-06-01 13:31:17 +0200359 fence_put(id->last_flush);
360 id->last_flush = fence;
Christian König41d9eb22016-03-01 16:46:18 +0100361 mutex_unlock(&adev->vm_manager.lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400362 }
Christian Königcffadc82016-03-01 13:34:49 +0100363
Christian Königd564a062016-03-01 15:51:53 +0100364 if (gds_switch_needed) {
Christian Königbcb1ba32016-03-08 15:40:11 +0100365 id->gds_base = gds_base;
366 id->gds_size = gds_size;
367 id->gws_base = gws_base;
368 id->gws_size = gws_size;
369 id->oa_base = oa_base;
370 id->oa_size = oa_size;
Christian Königcffadc82016-03-01 13:34:49 +0100371 amdgpu_ring_emit_gds_switch(ring, vm_id,
372 gds_base, gds_size,
373 gws_base, gws_size,
374 oa_base, oa_size);
Christian König971fe9a92016-03-01 15:09:25 +0100375 }
Christian König41d9eb22016-03-01 16:46:18 +0100376
377 return 0;
Christian König971fe9a92016-03-01 15:09:25 +0100378}
379
380/**
381 * amdgpu_vm_reset_id - reset VMID to zero
382 *
383 * @adev: amdgpu device structure
384 * @vm_id: vmid number to use
385 *
386 * Reset saved GDW, GWS and OA to force switch on next flush.
387 */
388void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
389{
Christian Königbcb1ba32016-03-08 15:40:11 +0100390 struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
Christian König971fe9a92016-03-01 15:09:25 +0100391
Christian Königbcb1ba32016-03-08 15:40:11 +0100392 id->gds_base = 0;
393 id->gds_size = 0;
394 id->gws_base = 0;
395 id->gws_size = 0;
396 id->oa_base = 0;
397 id->oa_size = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400398}
399
400/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400401 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
402 *
403 * @vm: requested vm
404 * @bo: requested buffer object
405 *
Christian König8843dbb2016-01-26 12:17:11 +0100406 * Find @bo inside the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400407 * Search inside the @bos vm list for the requested vm
408 * Returns the found bo_va or NULL if none is found
409 *
410 * Object has to be reserved!
411 */
412struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
413 struct amdgpu_bo *bo)
414{
415 struct amdgpu_bo_va *bo_va;
416
417 list_for_each_entry(bo_va, &bo->va, bo_list) {
418 if (bo_va->vm == vm) {
419 return bo_va;
420 }
421 }
422 return NULL;
423}
424
425/**
426 * amdgpu_vm_update_pages - helper to call the right asic function
427 *
428 * @adev: amdgpu_device pointer
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400429 * @vm_update_params: see amdgpu_vm_update_params definition
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400430 * @pe: addr of the page entry
431 * @addr: dst addr to write into pe
432 * @count: number of page entries to update
433 * @incr: increase next addr by incr bytes
434 * @flags: hw access flags
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400435 *
436 * Traces the parameters and calls the right asic functions
437 * to setup the page table using the DMA.
438 */
439static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400440 struct amdgpu_vm_update_params
441 *vm_update_params,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400442 uint64_t pe, uint64_t addr,
443 unsigned count, uint32_t incr,
Christian König9ab21462015-11-30 14:19:26 +0100444 uint32_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400445{
446 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);
447
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400448 if (vm_update_params->src) {
449 amdgpu_vm_copy_pte(adev, vm_update_params->ib,
450 pe, (vm_update_params->src + (addr >> 12) * 8), count);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400451
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400452 } else if (vm_update_params->pages_addr) {
453 amdgpu_vm_write_pte(adev, vm_update_params->ib,
454 vm_update_params->pages_addr,
455 pe, addr, count, incr, flags);
Christian Königb07c9d22015-11-30 13:26:07 +0100456
457 } else if (count < 3) {
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400458 amdgpu_vm_write_pte(adev, vm_update_params->ib, NULL, pe, addr,
Christian Königb07c9d22015-11-30 13:26:07 +0100459 count, incr, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400460
461 } else {
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400462 amdgpu_vm_set_pte_pde(adev, vm_update_params->ib, pe, addr,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400463 count, incr, flags);
464 }
465}
466
467/**
468 * amdgpu_vm_clear_bo - initially clear the page dir/table
469 *
470 * @adev: amdgpu_device pointer
471 * @bo: bo to clear
Chunming Zhouef9f0a82015-11-13 13:43:22 +0800472 *
473 * need to reserve bo first before calling it.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400474 */
475static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
Christian König2bd9ccf2016-02-01 12:53:58 +0100476 struct amdgpu_vm *vm,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400477 struct amdgpu_bo *bo)
478{
Christian König2d55e452016-02-08 17:37:38 +0100479 struct amdgpu_ring *ring;
Chunming Zhou4af9f072015-08-03 12:57:31 +0800480 struct fence *fence = NULL;
Christian Königd71518b2016-02-01 12:20:25 +0100481 struct amdgpu_job *job;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400482 struct amdgpu_vm_update_params vm_update_params;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400483 unsigned entries;
484 uint64_t addr;
485 int r;
486
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400487 memset(&vm_update_params, 0, sizeof(vm_update_params));
Christian König2d55e452016-02-08 17:37:38 +0100488 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
489
monk.liuca952612015-05-25 14:44:05 +0800490 r = reservation_object_reserve_shared(bo->tbo.resv);
491 if (r)
492 return r;
493
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400494 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
495 if (r)
Chunming Zhouef9f0a82015-11-13 13:43:22 +0800496 goto error;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400497
498 addr = amdgpu_bo_gpu_offset(bo);
499 entries = amdgpu_bo_size(bo) / 8;
500
Christian Königd71518b2016-02-01 12:20:25 +0100501 r = amdgpu_job_alloc_with_ib(adev, 64, &job);
502 if (r)
Chunming Zhouef9f0a82015-11-13 13:43:22 +0800503 goto error;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400504
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400505 vm_update_params.ib = &job->ibs[0];
506 amdgpu_vm_update_pages(adev, &vm_update_params, addr, 0, entries,
Christian Königd71518b2016-02-01 12:20:25 +0100507 0, 0);
508 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
509
510 WARN_ON(job->ibs[0].length_dw > 64);
Christian König2bd9ccf2016-02-01 12:53:58 +0100511 r = amdgpu_job_submit(job, ring, &vm->entity,
512 AMDGPU_FENCE_OWNER_VM, &fence);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400513 if (r)
514 goto error_free;
515
Christian Königd71518b2016-02-01 12:20:25 +0100516 amdgpu_bo_fence(bo, fence, true);
Chunming Zhou281b4222015-08-12 12:58:31 +0800517 fence_put(fence);
Chunming Zhoucadf97b2016-01-15 11:25:00 +0800518 return 0;
Chunming Zhouef9f0a82015-11-13 13:43:22 +0800519
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400520error_free:
Christian Königd71518b2016-02-01 12:20:25 +0100521 amdgpu_job_free(job);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400522
Chunming Zhouef9f0a82015-11-13 13:43:22 +0800523error:
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400524 return r;
525}
526
527/**
Christian Königb07c9d22015-11-30 13:26:07 +0100528 * amdgpu_vm_map_gart - Resolve gart mapping of addr
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400529 *
Christian Königb07c9d22015-11-30 13:26:07 +0100530 * @pages_addr: optional DMA address to use for lookup
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400531 * @addr: the unmapped addr
532 *
533 * Look up the physical address of the page that the pte resolves
Christian Königb07c9d22015-11-30 13:26:07 +0100534 * to and return the pointer for the page table entry.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400535 */
Christian Königb07c9d22015-11-30 13:26:07 +0100536uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400537{
538 uint64_t result;
539
Christian Königb07c9d22015-11-30 13:26:07 +0100540 if (pages_addr) {
541 /* page table offset */
542 result = pages_addr[addr >> PAGE_SHIFT];
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400543
Christian Königb07c9d22015-11-30 13:26:07 +0100544 /* in case cpu page size != gpu page size*/
545 result |= addr & (~PAGE_MASK);
546
547 } else {
548 /* No mapping required */
549 result = addr;
550 }
551
552 result &= 0xFFFFFFFFFFFFF000ULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400553
554 return result;
555}
556
557/**
558 * amdgpu_vm_update_pdes - make sure that page directory is valid
559 *
560 * @adev: amdgpu_device pointer
561 * @vm: requested vm
562 * @start: start of GPU address range
563 * @end: end of GPU address range
564 *
565 * Allocates new page tables if necessary
Christian König8843dbb2016-01-26 12:17:11 +0100566 * and updates the page directory.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400567 * Returns 0 for success, error for failure.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400568 */
569int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
570 struct amdgpu_vm *vm)
571{
Christian König2d55e452016-02-08 17:37:38 +0100572 struct amdgpu_ring *ring;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400573 struct amdgpu_bo *pd = vm->page_directory;
574 uint64_t pd_addr = amdgpu_bo_gpu_offset(pd);
575 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
576 uint64_t last_pde = ~0, last_pt = ~0;
577 unsigned count = 0, pt_idx, ndw;
Christian Königd71518b2016-02-01 12:20:25 +0100578 struct amdgpu_job *job;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400579 struct amdgpu_vm_update_params vm_update_params;
Chunming Zhou4af9f072015-08-03 12:57:31 +0800580 struct fence *fence = NULL;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800581
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400582 int r;
583
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400584 memset(&vm_update_params, 0, sizeof(vm_update_params));
Christian König2d55e452016-02-08 17:37:38 +0100585 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
586
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400587 /* padding, etc. */
588 ndw = 64;
589
590 /* assume the worst case */
591 ndw += vm->max_pde_used * 6;
592
Christian Königd71518b2016-02-01 12:20:25 +0100593 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
594 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400595 return r;
Christian Königd71518b2016-02-01 12:20:25 +0100596
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400597 vm_update_params.ib = &job->ibs[0];
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400598
599 /* walk over the address space and update the page directory */
600 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
Christian Königee1782c2015-12-11 21:01:23 +0100601 struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400602 uint64_t pde, pt;
603
604 if (bo == NULL)
605 continue;
606
607 pt = amdgpu_bo_gpu_offset(bo);
608 if (vm->page_tables[pt_idx].addr == pt)
609 continue;
610 vm->page_tables[pt_idx].addr = pt;
611
612 pde = pd_addr + pt_idx * 8;
613 if (((last_pde + 8 * count) != pde) ||
614 ((last_pt + incr * count) != pt)) {
615
616 if (count) {
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400617 amdgpu_vm_update_pages(adev, &vm_update_params,
Christian König9ab21462015-11-30 14:19:26 +0100618 last_pde, last_pt,
619 count, incr,
620 AMDGPU_PTE_VALID);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400621 }
622
623 count = 1;
624 last_pde = pde;
625 last_pt = pt;
626 } else {
627 ++count;
628 }
629 }
630
631 if (count)
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400632 amdgpu_vm_update_pages(adev, &vm_update_params,
633 last_pde, last_pt,
634 count, incr, AMDGPU_PTE_VALID);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400635
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400636 if (vm_update_params.ib->length_dw != 0) {
637 amdgpu_ring_pad_ib(ring, vm_update_params.ib);
Christian Könige86f9ce2016-02-08 12:13:05 +0100638 amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
639 AMDGPU_FENCE_OWNER_VM);
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400640 WARN_ON(vm_update_params.ib->length_dw > ndw);
Christian König2bd9ccf2016-02-01 12:53:58 +0100641 r = amdgpu_job_submit(job, ring, &vm->entity,
642 AMDGPU_FENCE_OWNER_VM, &fence);
Chunming Zhou4af9f072015-08-03 12:57:31 +0800643 if (r)
644 goto error_free;
Bas Nieuwenhuizen05906de2015-08-14 20:08:40 +0200645
Chunming Zhou4af9f072015-08-03 12:57:31 +0800646 amdgpu_bo_fence(pd, fence, true);
Bas Nieuwenhuizen05906de2015-08-14 20:08:40 +0200647 fence_put(vm->page_directory_fence);
648 vm->page_directory_fence = fence_get(fence);
Chunming Zhou281b4222015-08-12 12:58:31 +0800649 fence_put(fence);
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800650
Christian Königd71518b2016-02-01 12:20:25 +0100651 } else {
652 amdgpu_job_free(job);
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800653 }
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400654
655 return 0;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800656
657error_free:
Christian Königd71518b2016-02-01 12:20:25 +0100658 amdgpu_job_free(job);
Chunming Zhou4af9f072015-08-03 12:57:31 +0800659 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400660}
661
662/**
663 * amdgpu_vm_frag_ptes - add fragment information to PTEs
664 *
665 * @adev: amdgpu_device pointer
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400666 * @vm_update_params: see amdgpu_vm_update_params definition
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400667 * @pe_start: first PTE to handle
668 * @pe_end: last PTE to handle
669 * @addr: addr those PTEs should point to
670 * @flags: hw mapping flags
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400671 */
672static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400673 struct amdgpu_vm_update_params
674 *vm_update_params,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400675 uint64_t pe_start, uint64_t pe_end,
Christian König9ab21462015-11-30 14:19:26 +0100676 uint64_t addr, uint32_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400677{
678 /**
679 * The MC L1 TLB supports variable sized pages, based on a fragment
680 * field in the PTE. When this field is set to a non-zero value, page
681 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
682 * flags are considered valid for all PTEs within the fragment range
683 * and corresponding mappings are assumed to be physically contiguous.
684 *
685 * The L1 TLB can store a single PTE for the whole fragment,
686 * significantly increasing the space available for translation
687 * caching. This leads to large improvements in throughput when the
688 * TLB is under pressure.
689 *
690 * The L2 TLB distributes small and large fragments into two
691 * asymmetric partitions. The large fragment cache is significantly
692 * larger. Thus, we try to use large fragments wherever possible.
693 * Userspace can support this by aligning virtual base address and
694 * allocation size to the fragment size.
695 */
696
697 /* SI and newer are optimized for 64KB */
698 uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB;
699 uint64_t frag_align = 0x80;
700
701 uint64_t frag_start = ALIGN(pe_start, frag_align);
702 uint64_t frag_end = pe_end & ~(frag_align - 1);
703
704 unsigned count;
705
Christian König31f6c1f2016-01-26 12:37:49 +0100706 /* Abort early if there isn't anything to do */
707 if (pe_start == pe_end)
708 return;
709
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400710 /* system pages are non continuously */
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400711 if (vm_update_params->src || vm_update_params->pages_addr ||
712 !(flags & AMDGPU_PTE_VALID) || (frag_start >= frag_end)) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400713
714 count = (pe_end - pe_start) / 8;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400715 amdgpu_vm_update_pages(adev, vm_update_params, pe_start,
Christian König9ab21462015-11-30 14:19:26 +0100716 addr, count, AMDGPU_GPU_PAGE_SIZE,
717 flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400718 return;
719 }
720
721 /* handle the 4K area at the beginning */
722 if (pe_start != frag_start) {
723 count = (frag_start - pe_start) / 8;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400724 amdgpu_vm_update_pages(adev, vm_update_params, pe_start, addr,
Christian König9ab21462015-11-30 14:19:26 +0100725 count, AMDGPU_GPU_PAGE_SIZE, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400726 addr += AMDGPU_GPU_PAGE_SIZE * count;
727 }
728
729 /* handle the area in the middle */
730 count = (frag_end - frag_start) / 8;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400731 amdgpu_vm_update_pages(adev, vm_update_params, frag_start, addr, count,
Christian König9ab21462015-11-30 14:19:26 +0100732 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400733
734 /* handle the 4K area at the end */
735 if (frag_end != pe_end) {
736 addr += AMDGPU_GPU_PAGE_SIZE * count;
737 count = (pe_end - frag_end) / 8;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400738 amdgpu_vm_update_pages(adev, vm_update_params, frag_end, addr,
Christian König9ab21462015-11-30 14:19:26 +0100739 count, AMDGPU_GPU_PAGE_SIZE, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400740 }
741}
742
743/**
744 * amdgpu_vm_update_ptes - make sure that page tables are valid
745 *
746 * @adev: amdgpu_device pointer
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400747 * @vm_update_params: see amdgpu_vm_update_params definition
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400748 * @vm: requested vm
749 * @start: start of GPU address range
750 * @end: end of GPU address range
Alex Xie677131a2016-06-06 18:13:26 -0400751 * @dst: destination address to map to, the next dst inside the function
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400752 * @flags: mapping flags
753 *
Christian König8843dbb2016-01-26 12:17:11 +0100754 * Update the page tables in the range @start - @end.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400755 */
Christian Königa1e08d32016-01-26 11:40:46 +0100756static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400757 struct amdgpu_vm_update_params
758 *vm_update_params,
Christian Königa1e08d32016-01-26 11:40:46 +0100759 struct amdgpu_vm *vm,
Christian Königa1e08d32016-01-26 11:40:46 +0100760 uint64_t start, uint64_t end,
761 uint64_t dst, uint32_t flags)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400762{
Christian König31f6c1f2016-01-26 12:37:49 +0100763 const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
764
Alex Xie21718492016-06-06 18:21:09 -0400765 uint64_t cur_pe_start, cur_pe_end, cur_dst;
Alex Xie677131a2016-06-06 18:13:26 -0400766 uint64_t addr; /* next GPU address to be updated */
Alex Xie21718492016-06-06 18:21:09 -0400767 uint64_t pt_idx;
768 struct amdgpu_bo *pt;
769 unsigned nptes; /* next number of ptes to be updated */
770 uint64_t next_pe_start;
771
772 /* initialize the variables */
773 addr = start;
774 pt_idx = addr >> amdgpu_vm_block_size;
775 pt = vm->page_tables[pt_idx].entry.robj;
776
777 if ((addr & ~mask) == (end & ~mask))
778 nptes = end - addr;
779 else
780 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
781
782 cur_pe_start = amdgpu_bo_gpu_offset(pt);
783 cur_pe_start += (addr & mask) * 8;
784 cur_pe_end = cur_pe_start + 8 * nptes;
785 cur_dst = dst;
786
787 /* for next ptb*/
788 addr += nptes;
789 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400790
791 /* walk over the address space and update the page tables */
Alex Xie21718492016-06-06 18:21:09 -0400792 while (addr < end) {
793 pt_idx = addr >> amdgpu_vm_block_size;
794 pt = vm->page_tables[pt_idx].entry.robj;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400795
796 if ((addr & ~mask) == (end & ~mask))
797 nptes = end - addr;
798 else
799 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
800
Alex Xie677131a2016-06-06 18:13:26 -0400801 next_pe_start = amdgpu_bo_gpu_offset(pt);
802 next_pe_start += (addr & mask) * 8;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400803
Alex Xie3a6f8e02016-06-06 18:14:57 -0400804 if (cur_pe_end == next_pe_start) {
805 /* The next ptb is consecutive to current ptb.
806 * Don't call amdgpu_vm_frag_ptes now.
807 * Will update two ptbs together in future.
808 */
809 cur_pe_end += 8 * nptes;
810 } else {
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400811 amdgpu_vm_frag_ptes(adev, vm_update_params,
Alex Xie677131a2016-06-06 18:13:26 -0400812 cur_pe_start, cur_pe_end,
813 cur_dst, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400814
Alex Xie677131a2016-06-06 18:13:26 -0400815 cur_pe_start = next_pe_start;
816 cur_pe_end = next_pe_start + 8 * nptes;
817 cur_dst = dst;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400818 }
819
Alex Xie21718492016-06-06 18:21:09 -0400820 /* for next ptb*/
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400821 addr += nptes;
822 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
823 }
824
Alex Xie677131a2016-06-06 18:13:26 -0400825 amdgpu_vm_frag_ptes(adev, vm_update_params, cur_pe_start,
826 cur_pe_end, cur_dst, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400827}
828
829/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400830 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
831 *
832 * @adev: amdgpu_device pointer
Christian König3cabaa52016-06-06 10:17:58 +0200833 * @exclusive: fence we need to sync to
Christian Königfa3ab3c2016-03-18 21:00:35 +0100834 * @src: address where to copy page table entries from
835 * @pages_addr: DMA addresses to use for mapping
Christian Königa14faa62016-01-25 14:27:31 +0100836 * @vm: requested vm
837 * @start: start of mapped range
838 * @last: last mapped entry
839 * @flags: flags for the entries
840 * @addr: addr to set the area to
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400841 * @fence: optional resulting fence
842 *
Christian Königa14faa62016-01-25 14:27:31 +0100843 * Fill in the page table entries between @start and @last.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400844 * Returns 0 for success, -EINVAL for failure.
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400845 */
846static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
Christian König3cabaa52016-06-06 10:17:58 +0200847 struct fence *exclusive,
Christian Königfa3ab3c2016-03-18 21:00:35 +0100848 uint64_t src,
849 dma_addr_t *pages_addr,
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400850 struct amdgpu_vm *vm,
Christian Königa14faa62016-01-25 14:27:31 +0100851 uint64_t start, uint64_t last,
852 uint32_t flags, uint64_t addr,
853 struct fence **fence)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400854{
Christian König2d55e452016-02-08 17:37:38 +0100855 struct amdgpu_ring *ring;
Christian Königa1e08d32016-01-26 11:40:46 +0100856 void *owner = AMDGPU_FENCE_OWNER_VM;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400857 unsigned nptes, ncmds, ndw;
Christian Königd71518b2016-02-01 12:20:25 +0100858 struct amdgpu_job *job;
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400859 struct amdgpu_vm_update_params vm_update_params;
Chunming Zhou4af9f072015-08-03 12:57:31 +0800860 struct fence *f = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400861 int r;
862
Christian König2d55e452016-02-08 17:37:38 +0100863 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400864 memset(&vm_update_params, 0, sizeof(vm_update_params));
865 vm_update_params.src = src;
866 vm_update_params.pages_addr = pages_addr;
Christian König2d55e452016-02-08 17:37:38 +0100867
Christian Königa1e08d32016-01-26 11:40:46 +0100868 /* sync to everything on unmapping */
869 if (!(flags & AMDGPU_PTE_VALID))
870 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
871
Christian Königa14faa62016-01-25 14:27:31 +0100872 nptes = last - start + 1;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400873
874 /*
875 * reserve space for one command every (1 << BLOCK_SIZE)
876 * entries or 2k dwords (whatever is smaller)
877 */
878 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
879
880 /* padding, etc. */
881 ndw = 64;
882
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400883 if (vm_update_params.src) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400884 /* only copy commands needed */
885 ndw += ncmds * 7;
886
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400887 } else if (vm_update_params.pages_addr) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400888 /* header for write data commands */
889 ndw += ncmds * 4;
890
891 /* body of write data command */
892 ndw += nptes * 2;
893
894 } else {
895 /* set page commands needed */
896 ndw += ncmds * 10;
897
898 /* two extra commands for begin/end of fragment */
899 ndw += 2 * 10;
900 }
901
Christian Königd71518b2016-02-01 12:20:25 +0100902 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
903 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400904 return r;
Christian Königd71518b2016-02-01 12:20:25 +0100905
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400906 vm_update_params.ib = &job->ibs[0];
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800907
Christian König3cabaa52016-06-06 10:17:58 +0200908 r = amdgpu_sync_fence(adev, &job->sync, exclusive);
909 if (r)
910 goto error_free;
911
Christian Könige86f9ce2016-02-08 12:13:05 +0100912 r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
Christian Königa1e08d32016-01-26 11:40:46 +0100913 owner);
914 if (r)
915 goto error_free;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400916
Christian Königa1e08d32016-01-26 11:40:46 +0100917 r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
918 if (r)
919 goto error_free;
920
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400921 amdgpu_vm_update_ptes(adev, &vm_update_params, vm, start,
Christian Königfa3ab3c2016-03-18 21:00:35 +0100922 last + 1, addr, flags);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400923
Harish Kasiviswanathanf4833c42016-04-21 10:40:18 -0400924 amdgpu_ring_pad_ib(ring, vm_update_params.ib);
925 WARN_ON(vm_update_params.ib->length_dw > ndw);
Christian König2bd9ccf2016-02-01 12:53:58 +0100926 r = amdgpu_job_submit(job, ring, &vm->entity,
927 AMDGPU_FENCE_OWNER_VM, &f);
Chunming Zhou4af9f072015-08-03 12:57:31 +0800928 if (r)
929 goto error_free;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400930
Christian Königbf60efd2015-09-04 10:47:56 +0200931 amdgpu_bo_fence(vm->page_directory, f, true);
Chunming Zhou4af9f072015-08-03 12:57:31 +0800932 if (fence) {
933 fence_put(*fence);
934 *fence = fence_get(f);
935 }
Chunming Zhou281b4222015-08-12 12:58:31 +0800936 fence_put(f);
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400937 return 0;
Chunming Zhoud5fc5e82015-07-21 16:52:10 +0800938
939error_free:
Christian Königd71518b2016-02-01 12:20:25 +0100940 amdgpu_job_free(job);
Chunming Zhou4af9f072015-08-03 12:57:31 +0800941 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -0400942}
943
944/**
Christian Königa14faa62016-01-25 14:27:31 +0100945 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
946 *
947 * @adev: amdgpu_device pointer
Christian König3cabaa52016-06-06 10:17:58 +0200948 * @exclusive: fence we need to sync to
Christian König8358dce2016-03-30 10:50:25 +0200949 * @gtt_flags: flags as they are used for GTT
950 * @pages_addr: DMA addresses to use for mapping
Christian Königa14faa62016-01-25 14:27:31 +0100951 * @vm: requested vm
952 * @mapping: mapped range and flags to use for the update
953 * @addr: addr to set the area to
Christian König8358dce2016-03-30 10:50:25 +0200954 * @flags: HW flags for the mapping
Christian Königa14faa62016-01-25 14:27:31 +0100955 * @fence: optional resulting fence
956 *
957 * Split the mapping into smaller chunks so that each update fits
958 * into a SDMA IB.
959 * Returns 0 for success, -EINVAL for failure.
960 */
961static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
Christian König3cabaa52016-06-06 10:17:58 +0200962 struct fence *exclusive,
Christian Königa14faa62016-01-25 14:27:31 +0100963 uint32_t gtt_flags,
Christian König8358dce2016-03-30 10:50:25 +0200964 dma_addr_t *pages_addr,
Christian Königa14faa62016-01-25 14:27:31 +0100965 struct amdgpu_vm *vm,
966 struct amdgpu_bo_va_mapping *mapping,
Christian Königfa3ab3c2016-03-18 21:00:35 +0100967 uint32_t flags, uint64_t addr,
968 struct fence **fence)
Christian Königa14faa62016-01-25 14:27:31 +0100969{
970 const uint64_t max_size = 64ULL * 1024ULL * 1024ULL / AMDGPU_GPU_PAGE_SIZE;
971
Christian Königfa3ab3c2016-03-18 21:00:35 +0100972 uint64_t src = 0, start = mapping->it.start;
Christian Königa14faa62016-01-25 14:27:31 +0100973 int r;
974
975 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
976 * but in case of something, we filter the flags in first place
977 */
978 if (!(mapping->flags & AMDGPU_PTE_READABLE))
979 flags &= ~AMDGPU_PTE_READABLE;
980 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
981 flags &= ~AMDGPU_PTE_WRITEABLE;
982
983 trace_amdgpu_vm_bo_update(mapping);
984
Christian König8358dce2016-03-30 10:50:25 +0200985 if (pages_addr) {
Christian Königfa3ab3c2016-03-18 21:00:35 +0100986 if (flags == gtt_flags)
987 src = adev->gart.table_addr + (addr >> 12) * 8;
Christian Königfa3ab3c2016-03-18 21:00:35 +0100988 addr = 0;
989 }
Christian Königa14faa62016-01-25 14:27:31 +0100990 addr += mapping->offset;
991
Christian König8358dce2016-03-30 10:50:25 +0200992 if (!pages_addr || src)
Christian König3cabaa52016-06-06 10:17:58 +0200993 return amdgpu_vm_bo_update_mapping(adev, exclusive,
994 src, pages_addr, vm,
Christian Königa14faa62016-01-25 14:27:31 +0100995 start, mapping->it.last,
996 flags, addr, fence);
997
998 while (start != mapping->it.last + 1) {
999 uint64_t last;
1000
Felix Kuehlingfb29b572016-03-03 19:13:20 -05001001 last = min((uint64_t)mapping->it.last, start + max_size - 1);
Christian König3cabaa52016-06-06 10:17:58 +02001002 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1003 src, pages_addr, vm,
Christian Königa14faa62016-01-25 14:27:31 +01001004 start, last, flags, addr,
1005 fence);
1006 if (r)
1007 return r;
1008
1009 start = last + 1;
Felix Kuehlingfb29b572016-03-03 19:13:20 -05001010 addr += max_size * AMDGPU_GPU_PAGE_SIZE;
Christian Königa14faa62016-01-25 14:27:31 +01001011 }
1012
1013 return 0;
1014}
1015
1016/**
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001017 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1018 *
1019 * @adev: amdgpu_device pointer
1020 * @bo_va: requested BO and VM object
1021 * @mem: ttm mem
1022 *
1023 * Fill in the page table entries for @bo_va.
1024 * Returns 0 for success, -EINVAL for failure.
1025 *
1026 * Object have to be reserved and mutex must be locked!
1027 */
1028int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1029 struct amdgpu_bo_va *bo_va,
1030 struct ttm_mem_reg *mem)
1031{
1032 struct amdgpu_vm *vm = bo_va->vm;
1033 struct amdgpu_bo_va_mapping *mapping;
Christian König8358dce2016-03-30 10:50:25 +02001034 dma_addr_t *pages_addr = NULL;
Christian Königfa3ab3c2016-03-18 21:00:35 +01001035 uint32_t gtt_flags, flags;
Christian König3cabaa52016-06-06 10:17:58 +02001036 struct fence *exclusive;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001037 uint64_t addr;
1038 int r;
1039
1040 if (mem) {
Christian König8358dce2016-03-30 10:50:25 +02001041 struct ttm_dma_tt *ttm;
1042
Christian Königb7d698d2015-09-07 12:32:09 +02001043 addr = (u64)mem->start << PAGE_SHIFT;
Christian König9ab21462015-11-30 14:19:26 +01001044 switch (mem->mem_type) {
1045 case TTM_PL_TT:
Christian König8358dce2016-03-30 10:50:25 +02001046 ttm = container_of(bo_va->bo->tbo.ttm, struct
1047 ttm_dma_tt, ttm);
1048 pages_addr = ttm->dma_address;
Christian König9ab21462015-11-30 14:19:26 +01001049 break;
1050
1051 case TTM_PL_VRAM:
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001052 addr += adev->vm_manager.vram_base_offset;
Christian König9ab21462015-11-30 14:19:26 +01001053 break;
1054
1055 default:
1056 break;
1057 }
Christian König3cabaa52016-06-06 10:17:58 +02001058
1059 exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001060 } else {
1061 addr = 0;
Christian König3cabaa52016-06-06 10:17:58 +02001062 exclusive = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001063 }
1064
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001065 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
Christian Königfa3ab3c2016-03-18 21:00:35 +01001066 gtt_flags = (adev == bo_va->bo->adev) ? flags : 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001067
Christian König7fc11952015-07-30 11:53:42 +02001068 spin_lock(&vm->status_lock);
1069 if (!list_empty(&bo_va->vm_status))
1070 list_splice_init(&bo_va->valids, &bo_va->invalids);
1071 spin_unlock(&vm->status_lock);
1072
1073 list_for_each_entry(mapping, &bo_va->invalids, list) {
Christian König3cabaa52016-06-06 10:17:58 +02001074 r = amdgpu_vm_bo_split_mapping(adev, exclusive,
1075 gtt_flags, pages_addr, vm,
Christian König8358dce2016-03-30 10:50:25 +02001076 mapping, flags, addr,
1077 &bo_va->last_pt_update);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001078 if (r)
1079 return r;
1080 }
1081
Christian Königd6c10f62015-09-28 12:00:23 +02001082 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1083 list_for_each_entry(mapping, &bo_va->valids, list)
1084 trace_amdgpu_vm_bo_mapping(mapping);
1085
1086 list_for_each_entry(mapping, &bo_va->invalids, list)
1087 trace_amdgpu_vm_bo_mapping(mapping);
1088 }
1089
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001090 spin_lock(&vm->status_lock);
monk.liu6d1d0ef2015-08-14 13:36:41 +08001091 list_splice_init(&bo_va->invalids, &bo_va->valids);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001092 list_del_init(&bo_va->vm_status);
Christian König7fc11952015-07-30 11:53:42 +02001093 if (!mem)
1094 list_add(&bo_va->vm_status, &vm->cleared);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001095 spin_unlock(&vm->status_lock);
1096
1097 return 0;
1098}
1099
1100/**
1101 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1102 *
1103 * @adev: amdgpu_device pointer
1104 * @vm: requested vm
1105 *
1106 * Make sure all freed BOs are cleared in the PT.
1107 * Returns 0 for success.
1108 *
1109 * PTs have to be reserved and mutex must be locked!
1110 */
1111int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1112 struct amdgpu_vm *vm)
1113{
1114 struct amdgpu_bo_va_mapping *mapping;
1115 int r;
1116
1117 while (!list_empty(&vm->freed)) {
1118 mapping = list_first_entry(&vm->freed,
1119 struct amdgpu_bo_va_mapping, list);
1120 list_del(&mapping->list);
Christian Könige17841b2016-03-08 17:52:01 +01001121
Christian König3cabaa52016-06-06 10:17:58 +02001122 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
Christian Königfa3ab3c2016-03-18 21:00:35 +01001123 0, 0, NULL);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001124 kfree(mapping);
1125 if (r)
1126 return r;
1127
1128 }
1129 return 0;
1130
1131}
1132
1133/**
1134 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1135 *
1136 * @adev: amdgpu_device pointer
1137 * @vm: requested vm
1138 *
1139 * Make sure all invalidated BOs are cleared in the PT.
1140 * Returns 0 for success.
1141 *
1142 * PTs have to be reserved and mutex must be locked!
1143 */
1144int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
monk.liucfe2c972015-05-26 15:01:54 +08001145 struct amdgpu_vm *vm, struct amdgpu_sync *sync)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001146{
monk.liucfe2c972015-05-26 15:01:54 +08001147 struct amdgpu_bo_va *bo_va = NULL;
Christian König91e1a522015-07-06 22:06:40 +02001148 int r = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001149
1150 spin_lock(&vm->status_lock);
1151 while (!list_empty(&vm->invalidated)) {
1152 bo_va = list_first_entry(&vm->invalidated,
1153 struct amdgpu_bo_va, vm_status);
1154 spin_unlock(&vm->status_lock);
Christian König32b41ac2016-03-08 18:03:27 +01001155
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001156 r = amdgpu_vm_bo_update(adev, bo_va, NULL);
1157 if (r)
1158 return r;
1159
1160 spin_lock(&vm->status_lock);
1161 }
1162 spin_unlock(&vm->status_lock);
1163
monk.liucfe2c972015-05-26 15:01:54 +08001164 if (bo_va)
Chunming Zhoubb1e38a42015-08-03 18:19:38 +08001165 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
Christian König91e1a522015-07-06 22:06:40 +02001166
1167 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001168}
1169
1170/**
1171 * amdgpu_vm_bo_add - add a bo to a specific vm
1172 *
1173 * @adev: amdgpu_device pointer
1174 * @vm: requested vm
1175 * @bo: amdgpu buffer object
1176 *
Christian König8843dbb2016-01-26 12:17:11 +01001177 * Add @bo into the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001178 * Add @bo to the list of bos associated with the vm
1179 * Returns newly added bo_va or NULL for failure
1180 *
1181 * Object has to be reserved!
1182 */
1183struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1184 struct amdgpu_vm *vm,
1185 struct amdgpu_bo *bo)
1186{
1187 struct amdgpu_bo_va *bo_va;
1188
1189 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1190 if (bo_va == NULL) {
1191 return NULL;
1192 }
1193 bo_va->vm = vm;
1194 bo_va->bo = bo;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001195 bo_va->ref_count = 1;
1196 INIT_LIST_HEAD(&bo_va->bo_list);
Christian König7fc11952015-07-30 11:53:42 +02001197 INIT_LIST_HEAD(&bo_va->valids);
1198 INIT_LIST_HEAD(&bo_va->invalids);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001199 INIT_LIST_HEAD(&bo_va->vm_status);
Christian König32b41ac2016-03-08 18:03:27 +01001200
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001201 list_add_tail(&bo_va->bo_list, &bo->va);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001202
1203 return bo_va;
1204}
1205
1206/**
1207 * amdgpu_vm_bo_map - map bo inside a vm
1208 *
1209 * @adev: amdgpu_device pointer
1210 * @bo_va: bo_va to store the address
1211 * @saddr: where to map the BO
1212 * @offset: requested offset in the BO
1213 * @flags: attributes of pages (read/write/valid/etc.)
1214 *
1215 * Add a mapping of the BO at the specefied addr into the VM.
1216 * Returns 0 for success, error for failure.
1217 *
Chunming Zhou49b02b12015-11-13 14:18:38 +08001218 * Object has to be reserved and unreserved outside!
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001219 */
1220int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1221 struct amdgpu_bo_va *bo_va,
1222 uint64_t saddr, uint64_t offset,
1223 uint64_t size, uint32_t flags)
1224{
1225 struct amdgpu_bo_va_mapping *mapping;
1226 struct amdgpu_vm *vm = bo_va->vm;
1227 struct interval_tree_node *it;
1228 unsigned last_pfn, pt_idx;
1229 uint64_t eaddr;
1230 int r;
1231
Christian König0be52de2015-05-18 14:37:27 +02001232 /* validate the parameters */
1233 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
Chunming Zhou49b02b12015-11-13 14:18:38 +08001234 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
Christian König0be52de2015-05-18 14:37:27 +02001235 return -EINVAL;
Christian König0be52de2015-05-18 14:37:27 +02001236
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001237 /* make sure object fit at this offset */
Felix Kuehling005ae952015-11-23 17:43:48 -05001238 eaddr = saddr + size - 1;
Chunming Zhou49b02b12015-11-13 14:18:38 +08001239 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001240 return -EINVAL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001241
1242 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
Felix Kuehling005ae952015-11-23 17:43:48 -05001243 if (last_pfn >= adev->vm_manager.max_pfn) {
1244 dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001245 last_pfn, adev->vm_manager.max_pfn);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001246 return -EINVAL;
1247 }
1248
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001249 saddr /= AMDGPU_GPU_PAGE_SIZE;
1250 eaddr /= AMDGPU_GPU_PAGE_SIZE;
1251
Felix Kuehling005ae952015-11-23 17:43:48 -05001252 it = interval_tree_iter_first(&vm->va, saddr, eaddr);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001253 if (it) {
1254 struct amdgpu_bo_va_mapping *tmp;
1255 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1256 /* bo and tmp overlap, invalid addr */
1257 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1258 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1259 tmp->it.start, tmp->it.last + 1);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001260 r = -EINVAL;
Chunming Zhouf48b2652015-10-16 14:06:19 +08001261 goto error;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001262 }
1263
1264 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1265 if (!mapping) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001266 r = -ENOMEM;
Chunming Zhouf48b2652015-10-16 14:06:19 +08001267 goto error;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001268 }
1269
1270 INIT_LIST_HEAD(&mapping->list);
1271 mapping->it.start = saddr;
Felix Kuehling005ae952015-11-23 17:43:48 -05001272 mapping->it.last = eaddr;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001273 mapping->offset = offset;
1274 mapping->flags = flags;
1275
Christian König7fc11952015-07-30 11:53:42 +02001276 list_add(&mapping->list, &bo_va->invalids);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001277 interval_tree_insert(&mapping->it, &vm->va);
1278
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001279 /* Make sure the page tables are allocated */
1280 saddr >>= amdgpu_vm_block_size;
1281 eaddr >>= amdgpu_vm_block_size;
1282
1283 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1284
1285 if (eaddr > vm->max_pde_used)
1286 vm->max_pde_used = eaddr;
1287
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001288 /* walk over the address space and allocate the page tables */
1289 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
Christian Königbf60efd2015-09-04 10:47:56 +02001290 struct reservation_object *resv = vm->page_directory->tbo.resv;
Christian Königee1782c2015-12-11 21:01:23 +01001291 struct amdgpu_bo_list_entry *entry;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001292 struct amdgpu_bo *pt;
1293
Christian Königee1782c2015-12-11 21:01:23 +01001294 entry = &vm->page_tables[pt_idx].entry;
1295 if (entry->robj)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001296 continue;
1297
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001298 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1299 AMDGPU_GPU_PAGE_SIZE, true,
Alex Deucher857d9132015-08-27 00:14:16 -04001300 AMDGPU_GEM_DOMAIN_VRAM,
1301 AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
Christian Königbf60efd2015-09-04 10:47:56 +02001302 NULL, resv, &pt);
Chunming Zhou49b02b12015-11-13 14:18:38 +08001303 if (r)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001304 goto error_free;
Chunming Zhou49b02b12015-11-13 14:18:38 +08001305
Christian König82b9c552015-11-27 16:49:00 +01001306 /* Keep a reference to the page table to avoid freeing
1307 * them up in the wrong order.
1308 */
1309 pt->parent = amdgpu_bo_ref(vm->page_directory);
1310
Christian König2bd9ccf2016-02-01 12:53:58 +01001311 r = amdgpu_vm_clear_bo(adev, vm, pt);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001312 if (r) {
1313 amdgpu_bo_unref(&pt);
1314 goto error_free;
1315 }
1316
Christian Königee1782c2015-12-11 21:01:23 +01001317 entry->robj = pt;
Christian Königee1782c2015-12-11 21:01:23 +01001318 entry->priority = 0;
1319 entry->tv.bo = &entry->robj->tbo;
1320 entry->tv.shared = true;
Christian König2f568db2016-02-23 12:36:59 +01001321 entry->user_pages = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001322 vm->page_tables[pt_idx].addr = 0;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001323 }
1324
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001325 return 0;
1326
1327error_free:
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001328 list_del(&mapping->list);
1329 interval_tree_remove(&mapping->it, &vm->va);
Christian König93e3e432015-06-09 16:58:33 +02001330 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001331 kfree(mapping);
1332
Chunming Zhouf48b2652015-10-16 14:06:19 +08001333error:
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001334 return r;
1335}
1336
1337/**
1338 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1339 *
1340 * @adev: amdgpu_device pointer
1341 * @bo_va: bo_va to remove the address from
1342 * @saddr: where to the BO is mapped
1343 *
1344 * Remove a mapping of the BO at the specefied addr from the VM.
1345 * Returns 0 for success, error for failure.
1346 *
Chunming Zhou49b02b12015-11-13 14:18:38 +08001347 * Object has to be reserved and unreserved outside!
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001348 */
1349int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1350 struct amdgpu_bo_va *bo_va,
1351 uint64_t saddr)
1352{
1353 struct amdgpu_bo_va_mapping *mapping;
1354 struct amdgpu_vm *vm = bo_va->vm;
Christian König7fc11952015-07-30 11:53:42 +02001355 bool valid = true;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001356
Christian König6c7fc502015-06-05 20:56:17 +02001357 saddr /= AMDGPU_GPU_PAGE_SIZE;
Christian König32b41ac2016-03-08 18:03:27 +01001358
Christian König7fc11952015-07-30 11:53:42 +02001359 list_for_each_entry(mapping, &bo_va->valids, list) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001360 if (mapping->it.start == saddr)
1361 break;
1362 }
1363
Christian König7fc11952015-07-30 11:53:42 +02001364 if (&mapping->list == &bo_va->valids) {
1365 valid = false;
1366
1367 list_for_each_entry(mapping, &bo_va->invalids, list) {
1368 if (mapping->it.start == saddr)
1369 break;
1370 }
1371
Christian König32b41ac2016-03-08 18:03:27 +01001372 if (&mapping->list == &bo_va->invalids)
Christian König7fc11952015-07-30 11:53:42 +02001373 return -ENOENT;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001374 }
Christian König32b41ac2016-03-08 18:03:27 +01001375
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001376 list_del(&mapping->list);
1377 interval_tree_remove(&mapping->it, &vm->va);
Christian König93e3e432015-06-09 16:58:33 +02001378 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001379
Christian Könige17841b2016-03-08 17:52:01 +01001380 if (valid)
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001381 list_add(&mapping->list, &vm->freed);
Christian Könige17841b2016-03-08 17:52:01 +01001382 else
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001383 kfree(mapping);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001384
1385 return 0;
1386}
1387
1388/**
1389 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1390 *
1391 * @adev: amdgpu_device pointer
1392 * @bo_va: requested bo_va
1393 *
Christian König8843dbb2016-01-26 12:17:11 +01001394 * Remove @bo_va->bo from the requested vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001395 *
1396 * Object have to be reserved!
1397 */
1398void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1399 struct amdgpu_bo_va *bo_va)
1400{
1401 struct amdgpu_bo_va_mapping *mapping, *next;
1402 struct amdgpu_vm *vm = bo_va->vm;
1403
1404 list_del(&bo_va->bo_list);
1405
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001406 spin_lock(&vm->status_lock);
1407 list_del(&bo_va->vm_status);
1408 spin_unlock(&vm->status_lock);
1409
Christian König7fc11952015-07-30 11:53:42 +02001410 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001411 list_del(&mapping->list);
1412 interval_tree_remove(&mapping->it, &vm->va);
Christian König93e3e432015-06-09 16:58:33 +02001413 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
Christian König7fc11952015-07-30 11:53:42 +02001414 list_add(&mapping->list, &vm->freed);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001415 }
Christian König7fc11952015-07-30 11:53:42 +02001416 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1417 list_del(&mapping->list);
1418 interval_tree_remove(&mapping->it, &vm->va);
1419 kfree(mapping);
1420 }
Christian König32b41ac2016-03-08 18:03:27 +01001421
Chunming Zhoubb1e38a42015-08-03 18:19:38 +08001422 fence_put(bo_va->last_pt_update);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001423 kfree(bo_va);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001424}
1425
1426/**
1427 * amdgpu_vm_bo_invalidate - mark the bo as invalid
1428 *
1429 * @adev: amdgpu_device pointer
1430 * @vm: requested vm
1431 * @bo: amdgpu buffer object
1432 *
Christian König8843dbb2016-01-26 12:17:11 +01001433 * Mark @bo as invalid.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001434 */
1435void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1436 struct amdgpu_bo *bo)
1437{
1438 struct amdgpu_bo_va *bo_va;
1439
1440 list_for_each_entry(bo_va, &bo->va, bo_list) {
Christian König7fc11952015-07-30 11:53:42 +02001441 spin_lock(&bo_va->vm->status_lock);
1442 if (list_empty(&bo_va->vm_status))
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001443 list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
Christian König7fc11952015-07-30 11:53:42 +02001444 spin_unlock(&bo_va->vm->status_lock);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001445 }
1446}
1447
1448/**
1449 * amdgpu_vm_init - initialize a vm instance
1450 *
1451 * @adev: amdgpu_device pointer
1452 * @vm: requested vm
1453 *
Christian König8843dbb2016-01-26 12:17:11 +01001454 * Init @vm fields.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001455 */
1456int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1457{
1458 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1459 AMDGPU_VM_PTE_COUNT * 8);
Michel Dänzer9571e1d2016-01-19 17:59:46 +09001460 unsigned pd_size, pd_entries;
Christian König2d55e452016-02-08 17:37:38 +01001461 unsigned ring_instance;
1462 struct amdgpu_ring *ring;
Christian König2bd9ccf2016-02-01 12:53:58 +01001463 struct amd_sched_rq *rq;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001464 int i, r;
1465
Christian Königbcb1ba32016-03-08 15:40:11 +01001466 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1467 vm->ids[i] = NULL;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001468 vm->va = RB_ROOT;
Chunming Zhou031e2982016-04-25 10:19:13 +08001469 vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001470 spin_lock_init(&vm->status_lock);
1471 INIT_LIST_HEAD(&vm->invalidated);
Christian König7fc11952015-07-30 11:53:42 +02001472 INIT_LIST_HEAD(&vm->cleared);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001473 INIT_LIST_HEAD(&vm->freed);
Christian König20250212016-03-08 17:58:35 +01001474
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001475 pd_size = amdgpu_vm_directory_size(adev);
1476 pd_entries = amdgpu_vm_num_pdes(adev);
1477
1478 /* allocate page table array */
Michel Dänzer9571e1d2016-01-19 17:59:46 +09001479 vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001480 if (vm->page_tables == NULL) {
1481 DRM_ERROR("Cannot allocate memory for page table array\n");
1482 return -ENOMEM;
1483 }
1484
Christian König2bd9ccf2016-02-01 12:53:58 +01001485 /* create scheduler entity for page table updates */
Christian König2d55e452016-02-08 17:37:38 +01001486
1487 ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
1488 ring_instance %= adev->vm_manager.vm_pte_num_rings;
1489 ring = adev->vm_manager.vm_pte_rings[ring_instance];
Christian König2bd9ccf2016-02-01 12:53:58 +01001490 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
1491 r = amd_sched_entity_init(&ring->sched, &vm->entity,
1492 rq, amdgpu_sched_jobs);
1493 if (r)
1494 return r;
1495
Bas Nieuwenhuizen05906de2015-08-14 20:08:40 +02001496 vm->page_directory_fence = NULL;
1497
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001498 r = amdgpu_bo_create(adev, pd_size, align, true,
Alex Deucher857d9132015-08-27 00:14:16 -04001499 AMDGPU_GEM_DOMAIN_VRAM,
1500 AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
Christian König72d76682015-09-03 17:34:59 +02001501 NULL, NULL, &vm->page_directory);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001502 if (r)
Christian König2bd9ccf2016-02-01 12:53:58 +01001503 goto error_free_sched_entity;
1504
Chunming Zhouef9f0a82015-11-13 13:43:22 +08001505 r = amdgpu_bo_reserve(vm->page_directory, false);
Christian König2bd9ccf2016-02-01 12:53:58 +01001506 if (r)
1507 goto error_free_page_directory;
1508
1509 r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
Chunming Zhouef9f0a82015-11-13 13:43:22 +08001510 amdgpu_bo_unreserve(vm->page_directory);
Christian König2bd9ccf2016-02-01 12:53:58 +01001511 if (r)
1512 goto error_free_page_directory;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001513
1514 return 0;
Christian König2bd9ccf2016-02-01 12:53:58 +01001515
1516error_free_page_directory:
1517 amdgpu_bo_unref(&vm->page_directory);
1518 vm->page_directory = NULL;
1519
1520error_free_sched_entity:
1521 amd_sched_entity_fini(&ring->sched, &vm->entity);
1522
1523 return r;
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001524}
1525
1526/**
1527 * amdgpu_vm_fini - tear down a vm instance
1528 *
1529 * @adev: amdgpu_device pointer
1530 * @vm: requested vm
1531 *
Christian König8843dbb2016-01-26 12:17:11 +01001532 * Tear down @vm.
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001533 * Unbind the VM and remove all bos from the vm bo list
1534 */
1535void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1536{
1537 struct amdgpu_bo_va_mapping *mapping, *tmp;
1538 int i;
1539
Christian König2d55e452016-02-08 17:37:38 +01001540 amd_sched_entity_fini(vm->entity.sched, &vm->entity);
Christian König2bd9ccf2016-02-01 12:53:58 +01001541
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001542 if (!RB_EMPTY_ROOT(&vm->va)) {
1543 dev_err(adev->dev, "still active bo inside vm\n");
1544 }
1545 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1546 list_del(&mapping->list);
1547 interval_tree_remove(&mapping->it, &vm->va);
1548 kfree(mapping);
1549 }
1550 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1551 list_del(&mapping->list);
1552 kfree(mapping);
1553 }
1554
1555 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
Christian Königee1782c2015-12-11 21:01:23 +01001556 amdgpu_bo_unref(&vm->page_tables[i].entry.robj);
Michel Dänzer9571e1d2016-01-19 17:59:46 +09001557 drm_free_large(vm->page_tables);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001558
1559 amdgpu_bo_unref(&vm->page_directory);
Bas Nieuwenhuizen05906de2015-08-14 20:08:40 +02001560 fence_put(vm->page_directory_fence);
Alex Deucherd38ceaf2015-04-20 16:55:21 -04001561}
Christian Königea89f8c2015-11-15 20:52:06 +01001562
1563/**
Christian Königa9a78b32016-01-21 10:19:11 +01001564 * amdgpu_vm_manager_init - init the VM manager
1565 *
1566 * @adev: amdgpu_device pointer
1567 *
1568 * Initialize the VM manager structures
1569 */
1570void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1571{
1572 unsigned i;
1573
1574 INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1575
1576 /* skip over VMID 0, since it is the system VM */
Christian König971fe9a92016-03-01 15:09:25 +01001577 for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1578 amdgpu_vm_reset_id(adev, i);
Christian König832a9022016-02-15 12:33:02 +01001579 amdgpu_sync_create(&adev->vm_manager.ids[i].active);
Christian Königa9a78b32016-01-21 10:19:11 +01001580 list_add_tail(&adev->vm_manager.ids[i].list,
1581 &adev->vm_manager.ids_lru);
Christian König971fe9a92016-03-01 15:09:25 +01001582 }
Christian König2d55e452016-02-08 17:37:38 +01001583
Christian König1fbb2e92016-06-01 10:47:36 +02001584 adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
1585 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1586 adev->vm_manager.seqno[i] = 0;
1587
Christian König2d55e452016-02-08 17:37:38 +01001588 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
Christian Königb1c8a812016-05-04 10:34:03 +02001589 atomic64_set(&adev->vm_manager.client_counter, 0);
Christian Königa9a78b32016-01-21 10:19:11 +01001590}
1591
1592/**
Christian Königea89f8c2015-11-15 20:52:06 +01001593 * amdgpu_vm_manager_fini - cleanup VM manager
1594 *
1595 * @adev: amdgpu_device pointer
1596 *
1597 * Cleanup the VM manager and free resources.
1598 */
1599void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
1600{
1601 unsigned i;
1602
Christian Königbcb1ba32016-03-08 15:40:11 +01001603 for (i = 0; i < AMDGPU_NUM_VM; ++i) {
1604 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
1605
Christian König832a9022016-02-15 12:33:02 +01001606 fence_put(adev->vm_manager.ids[i].first);
1607 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
Christian Königbcb1ba32016-03-08 15:40:11 +01001608 fence_put(id->flushed_updates);
1609 }
Christian Königea89f8c2015-11-15 20:52:06 +01001610}