blob: fd109d3cae66a469607f797f06c7613822ab9eb5 [file] [log] [blame]
Jerome Glisse771fe6b2009-06-05 14:42:42 +02001/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
29#include <drm/radeon_drm.h>
Jerome Glisse771fe6b2009-06-05 14:42:42 +020030#include "radeon.h"
31#include "radeon_reg.h"
32
33/*
Alex Deucher03eec932012-07-17 14:02:39 -040034 * GART
35 * The GART (Graphics Aperture Remapping Table) is an aperture
36 * in the GPU's address space. System pages can be mapped into
37 * the aperture and look like contiguous pages from the GPU's
38 * perspective. A page table maps the pages in the aperture
39 * to the actual backing pages in system memory.
40 *
41 * Radeon GPUs support both an internal GART, as described above,
42 * and AGP. AGP works similarly, but the GART table is configured
43 * and maintained by the northbridge rather than the driver.
44 * Radeon hw has a separate AGP aperture that is programmed to
45 * point to the AGP aperture provided by the northbridge and the
46 * requests are passed through to the northbridge aperture.
47 * Both AGP and internal GART can be used at the same time, however
48 * that is not currently supported by the driver.
49 *
50 * This file handles the common internal GART management.
51 */
52
53/*
Jerome Glisse771fe6b2009-06-05 14:42:42 +020054 * Common GART table functions.
55 */
Alex Deucher03eec932012-07-17 14:02:39 -040056/**
57 * radeon_gart_table_ram_alloc - allocate system ram for gart page table
58 *
59 * @rdev: radeon_device pointer
60 *
61 * Allocate system memory for GART page table
62 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
63 * gart table to be in system memory.
64 * Returns 0 for success, -ENOMEM for failure.
65 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +020066int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
67{
68 void *ptr;
69
70 ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size,
71 &rdev->gart.table_addr);
72 if (ptr == NULL) {
73 return -ENOMEM;
74 }
75#ifdef CONFIG_X86
76 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
77 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
78 set_memory_uc((unsigned long)ptr,
79 rdev->gart.table_size >> PAGE_SHIFT);
80 }
81#endif
Jerome Glissec9a1be92011-11-03 11:16:49 -040082 rdev->gart.ptr = ptr;
83 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size);
Jerome Glisse771fe6b2009-06-05 14:42:42 +020084 return 0;
85}
86
Alex Deucher03eec932012-07-17 14:02:39 -040087/**
88 * radeon_gart_table_ram_free - free system ram for gart page table
89 *
90 * @rdev: radeon_device pointer
91 *
92 * Free system memory for GART page table
93 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
94 * gart table to be in system memory.
95 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +020096void radeon_gart_table_ram_free(struct radeon_device *rdev)
97{
Jerome Glissec9a1be92011-11-03 11:16:49 -040098 if (rdev->gart.ptr == NULL) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +020099 return;
100 }
101#ifdef CONFIG_X86
102 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
103 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400104 set_memory_wb((unsigned long)rdev->gart.ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200105 rdev->gart.table_size >> PAGE_SHIFT);
106 }
107#endif
108 pci_free_consistent(rdev->pdev, rdev->gart.table_size,
Jerome Glissec9a1be92011-11-03 11:16:49 -0400109 (void *)rdev->gart.ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200110 rdev->gart.table_addr);
Jerome Glissec9a1be92011-11-03 11:16:49 -0400111 rdev->gart.ptr = NULL;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200112 rdev->gart.table_addr = 0;
113}
114
Alex Deucher03eec932012-07-17 14:02:39 -0400115/**
116 * radeon_gart_table_vram_alloc - allocate vram for gart page table
117 *
118 * @rdev: radeon_device pointer
119 *
120 * Allocate video memory for GART page table
121 * (pcie r4xx, r5xx+). These asics require the
122 * gart table to be in video memory.
123 * Returns 0 for success, error for failure.
124 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200125int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
126{
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200127 int r;
128
Jerome Glissec9a1be92011-11-03 11:16:49 -0400129 if (rdev->gart.robj == NULL) {
Daniel Vetter441921d2011-02-18 17:59:16 +0100130 r = radeon_bo_create(rdev, rdev->gart.table_size,
Alex Deucher268b2512010-11-17 19:00:26 -0500131 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
Alex Deucher40f5cf92012-05-10 18:33:13 -0400132 NULL, &rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200133 if (r) {
134 return r;
135 }
136 }
Jerome Glisse4aac0472009-09-14 18:29:49 +0200137 return 0;
138}
139
Alex Deucher03eec932012-07-17 14:02:39 -0400140/**
141 * radeon_gart_table_vram_pin - pin gart page table in vram
142 *
143 * @rdev: radeon_device pointer
144 *
145 * Pin the GART page table in vram so it will not be moved
146 * by the memory manager (pcie r4xx, r5xx+). These asics require the
147 * gart table to be in video memory.
148 * Returns 0 for success, error for failure.
149 */
Jerome Glisse4aac0472009-09-14 18:29:49 +0200150int radeon_gart_table_vram_pin(struct radeon_device *rdev)
151{
152 uint64_t gpu_addr;
153 int r;
154
Jerome Glissec9a1be92011-11-03 11:16:49 -0400155 r = radeon_bo_reserve(rdev->gart.robj, false);
Jerome Glisse4c788672009-11-20 14:29:23 +0100156 if (unlikely(r != 0))
157 return r;
Jerome Glissec9a1be92011-11-03 11:16:49 -0400158 r = radeon_bo_pin(rdev->gart.robj,
Jerome Glisse4c788672009-11-20 14:29:23 +0100159 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200160 if (r) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400161 radeon_bo_unreserve(rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200162 return r;
163 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400164 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
Jerome Glisse4c788672009-11-20 14:29:23 +0100165 if (r)
Jerome Glissec9a1be92011-11-03 11:16:49 -0400166 radeon_bo_unpin(rdev->gart.robj);
167 radeon_bo_unreserve(rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200168 rdev->gart.table_addr = gpu_addr;
Jerome Glisse4c788672009-11-20 14:29:23 +0100169 return r;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200170}
171
Alex Deucher03eec932012-07-17 14:02:39 -0400172/**
173 * radeon_gart_table_vram_unpin - unpin gart page table in vram
174 *
175 * @rdev: radeon_device pointer
176 *
177 * Unpin the GART page table in vram (pcie r4xx, r5xx+).
178 * These asics require the gart table to be in video memory.
179 */
Jerome Glissec9a1be92011-11-03 11:16:49 -0400180void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200181{
Jerome Glisse4c788672009-11-20 14:29:23 +0100182 int r;
183
Jerome Glissec9a1be92011-11-03 11:16:49 -0400184 if (rdev->gart.robj == NULL) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200185 return;
186 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400187 r = radeon_bo_reserve(rdev->gart.robj, false);
Jerome Glisse4c788672009-11-20 14:29:23 +0100188 if (likely(r == 0)) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400189 radeon_bo_kunmap(rdev->gart.robj);
190 radeon_bo_unpin(rdev->gart.robj);
191 radeon_bo_unreserve(rdev->gart.robj);
192 rdev->gart.ptr = NULL;
Jerome Glisse4c788672009-11-20 14:29:23 +0100193 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400194}
195
Alex Deucher03eec932012-07-17 14:02:39 -0400196/**
197 * radeon_gart_table_vram_free - free gart page table vram
198 *
199 * @rdev: radeon_device pointer
200 *
201 * Free the video memory used for the GART page table
202 * (pcie r4xx, r5xx+). These asics require the gart table to
203 * be in video memory.
204 */
Jerome Glissec9a1be92011-11-03 11:16:49 -0400205void radeon_gart_table_vram_free(struct radeon_device *rdev)
206{
207 if (rdev->gart.robj == NULL) {
208 return;
209 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400210 radeon_bo_unref(&rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200211}
212
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200213/*
214 * Common gart functions.
215 */
Alex Deucher03eec932012-07-17 14:02:39 -0400216/**
217 * radeon_gart_unbind - unbind pages from the gart page table
218 *
219 * @rdev: radeon_device pointer
220 * @offset: offset into the GPU's gart aperture
221 * @pages: number of pages to unbind
222 *
223 * Unbinds the requested pages from the gart page table and
224 * replaces them with the dummy page (all asics).
225 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200226void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
227 int pages)
228{
229 unsigned t;
230 unsigned p;
231 int i, j;
Dave Airlie82568562010-02-05 16:00:07 +1000232 u64 page_base;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200233
234 if (!rdev->gart.ready) {
Tormod Voldenfcf4de52011-08-31 21:54:07 +0000235 WARN(1, "trying to unbind memory from uninitialized GART !\n");
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200236 return;
237 }
Matt Turnera77f1712009-10-14 00:34:41 -0400238 t = offset / RADEON_GPU_PAGE_SIZE;
239 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200240 for (i = 0; i < pages; i++, p++) {
241 if (rdev->gart.pages[p]) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200242 rdev->gart.pages[p] = NULL;
Dave Airlie82568562010-02-05 16:00:07 +1000243 rdev->gart.pages_addr[p] = rdev->dummy_page.addr;
244 page_base = rdev->gart.pages_addr[p];
Matt Turnera77f1712009-10-14 00:34:41 -0400245 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400246 if (rdev->gart.ptr) {
247 radeon_gart_set_page(rdev, t, page_base);
248 }
Dave Airlie82568562010-02-05 16:00:07 +1000249 page_base += RADEON_GPU_PAGE_SIZE;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200250 }
251 }
252 }
253 mb();
254 radeon_gart_tlb_flush(rdev);
255}
256
Alex Deucher03eec932012-07-17 14:02:39 -0400257/**
258 * radeon_gart_bind - bind pages into the gart page table
259 *
260 * @rdev: radeon_device pointer
261 * @offset: offset into the GPU's gart aperture
262 * @pages: number of pages to bind
263 * @pagelist: pages to bind
264 * @dma_addr: DMA addresses of pages
265 *
266 * Binds the requested pages to the gart page table
267 * (all asics).
268 * Returns 0 for success, -EINVAL for failure.
269 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200270int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
Konrad Rzeszutek Wilkc39d3512010-12-02 11:04:29 -0500271 int pages, struct page **pagelist, dma_addr_t *dma_addr)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200272{
273 unsigned t;
274 unsigned p;
275 uint64_t page_base;
276 int i, j;
277
278 if (!rdev->gart.ready) {
Tormod Voldenfcf4de52011-08-31 21:54:07 +0000279 WARN(1, "trying to bind memory to uninitialized GART !\n");
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200280 return -EINVAL;
281 }
Matt Turnera77f1712009-10-14 00:34:41 -0400282 t = offset / RADEON_GPU_PAGE_SIZE;
283 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200284
285 for (i = 0; i < pages; i++, p++) {
Konrad Rzeszutek Wilkc52494f2011-10-17 17:15:08 -0400286 rdev->gart.pages_addr[p] = dma_addr[i];
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200287 rdev->gart.pages[p] = pagelist[i];
Jerome Glissec9a1be92011-11-03 11:16:49 -0400288 if (rdev->gart.ptr) {
289 page_base = rdev->gart.pages_addr[p];
290 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
291 radeon_gart_set_page(rdev, t, page_base);
292 page_base += RADEON_GPU_PAGE_SIZE;
293 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200294 }
295 }
296 mb();
297 radeon_gart_tlb_flush(rdev);
298 return 0;
299}
300
Alex Deucher03eec932012-07-17 14:02:39 -0400301/**
302 * radeon_gart_restore - bind all pages in the gart page table
303 *
304 * @rdev: radeon_device pointer
305 *
306 * Binds all pages in the gart page table (all asics).
307 * Used to rebuild the gart table on device startup or resume.
308 */
Dave Airlie82568562010-02-05 16:00:07 +1000309void radeon_gart_restore(struct radeon_device *rdev)
310{
311 int i, j, t;
312 u64 page_base;
313
Jerome Glissec9a1be92011-11-03 11:16:49 -0400314 if (!rdev->gart.ptr) {
315 return;
316 }
Dave Airlie82568562010-02-05 16:00:07 +1000317 for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) {
318 page_base = rdev->gart.pages_addr[i];
319 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
320 radeon_gart_set_page(rdev, t, page_base);
321 page_base += RADEON_GPU_PAGE_SIZE;
322 }
323 }
324 mb();
325 radeon_gart_tlb_flush(rdev);
326}
327
Alex Deucher03eec932012-07-17 14:02:39 -0400328/**
329 * radeon_gart_init - init the driver info for managing the gart
330 *
331 * @rdev: radeon_device pointer
332 *
333 * Allocate the dummy page and init the gart driver info (all asics).
334 * Returns 0 for success, error for failure.
335 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200336int radeon_gart_init(struct radeon_device *rdev)
337{
Dave Airlie82568562010-02-05 16:00:07 +1000338 int r, i;
339
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200340 if (rdev->gart.pages) {
341 return 0;
342 }
Matt Turnera77f1712009-10-14 00:34:41 -0400343 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */
344 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200345 DRM_ERROR("Page size is smaller than GPU page size!\n");
346 return -EINVAL;
347 }
Dave Airlie82568562010-02-05 16:00:07 +1000348 r = radeon_dummy_page_init(rdev);
349 if (r)
350 return r;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200351 /* Compute table size */
352 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
Matt Turnera77f1712009-10-14 00:34:41 -0400353 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200354 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
355 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
356 /* Allocate pages table */
Christian König59240ee2012-10-23 15:53:17 +0200357 rdev->gart.pages = vzalloc(sizeof(void *) * rdev->gart.num_cpu_pages);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200358 if (rdev->gart.pages == NULL) {
359 radeon_gart_fini(rdev);
360 return -ENOMEM;
361 }
Christian König59240ee2012-10-23 15:53:17 +0200362 rdev->gart.pages_addr = vzalloc(sizeof(dma_addr_t) *
363 rdev->gart.num_cpu_pages);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200364 if (rdev->gart.pages_addr == NULL) {
365 radeon_gart_fini(rdev);
366 return -ENOMEM;
367 }
Dave Airlie82568562010-02-05 16:00:07 +1000368 /* set GART entry to point to the dummy page by default */
369 for (i = 0; i < rdev->gart.num_cpu_pages; i++) {
370 rdev->gart.pages_addr[i] = rdev->dummy_page.addr;
371 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200372 return 0;
373}
374
Alex Deucher03eec932012-07-17 14:02:39 -0400375/**
376 * radeon_gart_fini - tear down the driver info for managing the gart
377 *
378 * @rdev: radeon_device pointer
379 *
380 * Tear down the gart driver info and free the dummy page (all asics).
381 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200382void radeon_gart_fini(struct radeon_device *rdev)
383{
384 if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) {
385 /* unbind pages */
386 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
387 }
388 rdev->gart.ready = false;
Christian König59240ee2012-10-23 15:53:17 +0200389 vfree(rdev->gart.pages);
390 vfree(rdev->gart.pages_addr);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200391 rdev->gart.pages = NULL;
392 rdev->gart.pages_addr = NULL;
Alex Deucher92656d72011-04-12 13:32:13 -0400393
394 radeon_dummy_page_fini(rdev);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200395}
Jerome Glisse721604a2012-01-05 22:11:05 -0500396
397/*
Alex Deucher09db8642012-07-17 14:02:40 -0400398 * GPUVM
399 * GPUVM is similar to the legacy gart on older asics, however
400 * rather than there being a single global gart table
401 * for the entire GPU, there are multiple VM page tables active
402 * at any given time. The VM page tables can contain a mix
403 * vram pages and system memory pages and system memory pages
404 * can be mapped as snooped (cached system pages) or unsnooped
405 * (uncached system pages).
406 * Each VM has an ID associated with it and there is a page table
407 * associated with each VMID. When execting a command buffer,
408 * the kernel tells the the ring what VMID to use for that command
409 * buffer. VMIDs are allocated dynamically as commands are submitted.
410 * The userspace drivers maintain their own address space and the kernel
411 * sets up their pages tables accordingly when they submit their
412 * command buffers and a VMID is assigned.
413 * Cayman/Trinity support up to 8 active VMs at any given time;
414 * SI supports 16.
415 */
416
417/*
Jerome Glisse721604a2012-01-05 22:11:05 -0500418 * vm helpers
419 *
420 * TODO bind a default page at vm initialization for default address
421 */
Christian Königc6105f22012-07-05 14:32:00 +0200422
Alex Deucher09db8642012-07-17 14:02:40 -0400423/**
Christian König90a51a32012-10-09 13:31:17 +0200424 * radeon_vm_num_pde - return the number of page directory entries
425 *
426 * @rdev: radeon_device pointer
427 *
428 * Calculate the number of page directory entries (cayman+).
429 */
430static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)
431{
432 return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE;
433}
434
435/**
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200436 * radeon_vm_directory_size - returns the size of the page directory in bytes
437 *
438 * @rdev: radeon_device pointer
439 *
440 * Calculate the size of the page directory in bytes (cayman+).
441 */
442static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
443{
Christian König90a51a32012-10-09 13:31:17 +0200444 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200445}
446
447/**
Alex Deucher09db8642012-07-17 14:02:40 -0400448 * radeon_vm_manager_init - init the vm manager
449 *
450 * @rdev: radeon_device pointer
451 *
452 * Init the vm manager (cayman+).
453 * Returns 0 for success, error for failure.
454 */
Jerome Glisse721604a2012-01-05 22:11:05 -0500455int radeon_vm_manager_init(struct radeon_device *rdev)
456{
Christian Königc6105f22012-07-05 14:32:00 +0200457 struct radeon_vm *vm;
458 struct radeon_bo_va *bo_va;
Jerome Glisse721604a2012-01-05 22:11:05 -0500459 int r;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200460 unsigned size;
Jerome Glisse721604a2012-01-05 22:11:05 -0500461
Christian Königc6105f22012-07-05 14:32:00 +0200462 if (!rdev->vm_manager.enabled) {
Dave Airliee6b0b6a2012-07-20 00:53:28 -0400463 /* allocate enough for 2 full VM pts */
Christian König90a51a32012-10-09 13:31:17 +0200464 size = radeon_vm_directory_size(rdev);
465 size += rdev->vm_manager.max_pfn * 8;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200466 size *= 2;
Christian Königc6105f22012-07-05 14:32:00 +0200467 r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager,
Alex Deucher3e3e53f2013-07-18 13:11:56 -0400468 RADEON_GPU_PAGE_ALIGN(size),
Alex Deucher1c011032013-07-12 15:56:02 -0400469 RADEON_VM_PTB_ALIGN_SIZE,
Christian Königc6105f22012-07-05 14:32:00 +0200470 RADEON_GEM_DOMAIN_VRAM);
471 if (r) {
472 dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n",
473 (rdev->vm_manager.max_pfn * 8) >> 10);
474 return r;
475 }
Alex Deucher67e915e2012-01-06 09:38:15 -0500476
Christian König05b07142012-08-06 20:21:10 +0200477 r = radeon_asic_vm_init(rdev);
Christian Königc6105f22012-07-05 14:32:00 +0200478 if (r)
479 return r;
Christian König089a7862012-08-11 11:54:05 +0200480
Alex Deucher67e915e2012-01-06 09:38:15 -0500481 rdev->vm_manager.enabled = true;
482
Christian Königc6105f22012-07-05 14:32:00 +0200483 r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager);
484 if (r)
485 return r;
486 }
487
488 /* restore page table */
489 list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) {
Christian König90a51a32012-10-09 13:31:17 +0200490 if (vm->page_directory == NULL)
Christian Königc6105f22012-07-05 14:32:00 +0200491 continue;
492
493 list_for_each_entry(bo_va, &vm->va, vm_list) {
Christian Königc6105f22012-07-05 14:32:00 +0200494 bo_va->valid = false;
Christian Königc6105f22012-07-05 14:32:00 +0200495 }
496 }
497 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500498}
499
Alex Deucher09db8642012-07-17 14:02:40 -0400500/**
Christian Königddf03f52012-08-09 20:02:28 +0200501 * radeon_vm_free_pt - free the page table for a specific vm
Alex Deucher09db8642012-07-17 14:02:40 -0400502 *
503 * @rdev: radeon_device pointer
504 * @vm: vm to unbind
505 *
Christian Königddf03f52012-08-09 20:02:28 +0200506 * Free the page table of a specific vm (cayman+).
507 *
508 * Global and local mutex must be lock!
Alex Deucher09db8642012-07-17 14:02:40 -0400509 */
Christian Königddf03f52012-08-09 20:02:28 +0200510static void radeon_vm_free_pt(struct radeon_device *rdev,
Jerome Glisse721604a2012-01-05 22:11:05 -0500511 struct radeon_vm *vm)
512{
513 struct radeon_bo_va *bo_va;
Christian König90a51a32012-10-09 13:31:17 +0200514 int i;
Jerome Glisse721604a2012-01-05 22:11:05 -0500515
Christian König90a51a32012-10-09 13:31:17 +0200516 if (!vm->page_directory)
Jerome Glisse721604a2012-01-05 22:11:05 -0500517 return;
Jerome Glisse721604a2012-01-05 22:11:05 -0500518
Jerome Glisse721604a2012-01-05 22:11:05 -0500519 list_del_init(&vm->list);
Christian König90a51a32012-10-09 13:31:17 +0200520 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
Jerome Glisse721604a2012-01-05 22:11:05 -0500521
522 list_for_each_entry(bo_va, &vm->va, vm_list) {
523 bo_va->valid = false;
524 }
Christian König90a51a32012-10-09 13:31:17 +0200525
526 if (vm->page_tables == NULL)
527 return;
528
529 for (i = 0; i < radeon_vm_num_pdes(rdev); i++)
530 radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence);
531
532 kfree(vm->page_tables);
Jerome Glisse721604a2012-01-05 22:11:05 -0500533}
534
Alex Deucher09db8642012-07-17 14:02:40 -0400535/**
536 * radeon_vm_manager_fini - tear down the vm manager
537 *
538 * @rdev: radeon_device pointer
539 *
540 * Tear down the VM manager (cayman+).
541 */
Jerome Glisse721604a2012-01-05 22:11:05 -0500542void radeon_vm_manager_fini(struct radeon_device *rdev)
543{
Jerome Glisse721604a2012-01-05 22:11:05 -0500544 struct radeon_vm *vm, *tmp;
Christian Königee60e292012-08-09 16:21:08 +0200545 int i;
Jerome Glisse721604a2012-01-05 22:11:05 -0500546
Christian Königc6105f22012-07-05 14:32:00 +0200547 if (!rdev->vm_manager.enabled)
548 return;
549
Christian König36ff39c2012-05-09 10:07:08 +0200550 mutex_lock(&rdev->vm_manager.lock);
Christian Königddf03f52012-08-09 20:02:28 +0200551 /* free all allocated page tables */
Jerome Glisse721604a2012-01-05 22:11:05 -0500552 list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) {
Christian Königddf03f52012-08-09 20:02:28 +0200553 mutex_lock(&vm->mutex);
554 radeon_vm_free_pt(rdev, vm);
555 mutex_unlock(&vm->mutex);
Jerome Glisse721604a2012-01-05 22:11:05 -0500556 }
Christian Königee60e292012-08-09 16:21:08 +0200557 for (i = 0; i < RADEON_NUM_VM; ++i) {
558 radeon_fence_unref(&rdev->vm_manager.active[i]);
559 }
Christian König05b07142012-08-06 20:21:10 +0200560 radeon_asic_vm_fini(rdev);
Christian König36ff39c2012-05-09 10:07:08 +0200561 mutex_unlock(&rdev->vm_manager.lock);
Christian Königc6105f22012-07-05 14:32:00 +0200562
563 radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager);
564 radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager);
565 rdev->vm_manager.enabled = false;
Jerome Glisse721604a2012-01-05 22:11:05 -0500566}
567
Alex Deucher09db8642012-07-17 14:02:40 -0400568/**
Christian König90a51a32012-10-09 13:31:17 +0200569 * radeon_vm_evict - evict page table to make room for new one
570 *
571 * @rdev: radeon_device pointer
572 * @vm: VM we want to allocate something for
573 *
574 * Evict a VM from the lru, making sure that it isn't @vm. (cayman+).
575 * Returns 0 for success, -ENOMEM for failure.
576 *
577 * Global and local mutex must be locked!
578 */
Alex Deucher1518d7f2012-10-17 12:42:13 -0400579static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm)
Christian König90a51a32012-10-09 13:31:17 +0200580{
581 struct radeon_vm *vm_evict;
582
583 if (list_empty(&rdev->vm_manager.lru_vm))
584 return -ENOMEM;
585
586 vm_evict = list_first_entry(&rdev->vm_manager.lru_vm,
587 struct radeon_vm, list);
588 if (vm_evict == vm)
589 return -ENOMEM;
590
591 mutex_lock(&vm_evict->mutex);
592 radeon_vm_free_pt(rdev, vm_evict);
593 mutex_unlock(&vm_evict->mutex);
594 return 0;
595}
596
597/**
Christian Königddf03f52012-08-09 20:02:28 +0200598 * radeon_vm_alloc_pt - allocates a page table for a VM
Alex Deucher09db8642012-07-17 14:02:40 -0400599 *
600 * @rdev: radeon_device pointer
601 * @vm: vm to bind
602 *
Christian Königddf03f52012-08-09 20:02:28 +0200603 * Allocate a page table for the requested vm (cayman+).
Alex Deucher09db8642012-07-17 14:02:40 -0400604 * Returns 0 for success, error for failure.
Christian Königddf03f52012-08-09 20:02:28 +0200605 *
606 * Global and local mutex must be locked!
Alex Deucher09db8642012-07-17 14:02:40 -0400607 */
Christian Königddf03f52012-08-09 20:02:28 +0200608int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
Jerome Glisse721604a2012-01-05 22:11:05 -0500609{
Christian König90a51a32012-10-09 13:31:17 +0200610 unsigned pd_size, pts_size;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200611 u64 *pd_addr;
Christian König90a51a32012-10-09 13:31:17 +0200612 int r;
Jerome Glisse721604a2012-01-05 22:11:05 -0500613
614 if (vm == NULL) {
615 return -EINVAL;
616 }
617
Christian König90a51a32012-10-09 13:31:17 +0200618 if (vm->page_directory != NULL) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500619 return 0;
620 }
621
622retry:
Alex Deucher3e3e53f2013-07-18 13:11:56 -0400623 pd_size = radeon_vm_directory_size(rdev);
Christian König90a51a32012-10-09 13:31:17 +0200624 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
625 &vm->page_directory, pd_size,
Alex Deucher1c011032013-07-12 15:56:02 -0400626 RADEON_VM_PTB_ALIGN_SIZE, false);
Christian Königddf03f52012-08-09 20:02:28 +0200627 if (r == -ENOMEM) {
Christian König90a51a32012-10-09 13:31:17 +0200628 r = radeon_vm_evict(rdev, vm);
629 if (r)
Jerome Glisse721604a2012-01-05 22:11:05 -0500630 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -0500631 goto retry;
Jerome Glisse721604a2012-01-05 22:11:05 -0500632
Christian Königddf03f52012-08-09 20:02:28 +0200633 } else if (r) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500634 return r;
635 }
Christian Königddf03f52012-08-09 20:02:28 +0200636
Christian König90a51a32012-10-09 13:31:17 +0200637 vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory);
638
639 /* Initially clear the page directory */
640 pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory);
641 memset(pd_addr, 0, pd_size);
642
643 pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *);
644 vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
645
646 if (vm->page_tables == NULL) {
647 DRM_ERROR("Cannot allocate memory for page table array\n");
648 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
649 return -ENOMEM;
650 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500651
Christian Königd72d43c2012-10-09 13:31:18 +0200652 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500653}
654
Christian Königee60e292012-08-09 16:21:08 +0200655/**
Christian König13e55c32012-10-09 13:31:19 +0200656 * radeon_vm_add_to_lru - add VMs page table to LRU list
657 *
658 * @rdev: radeon_device pointer
659 * @vm: vm to add to LRU
660 *
661 * Add the allocated page table to the LRU list (cayman+).
662 *
663 * Global mutex must be locked!
664 */
665void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm)
666{
667 list_del_init(&vm->list);
668 list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
669}
670
671/**
Christian Königee60e292012-08-09 16:21:08 +0200672 * radeon_vm_grab_id - allocate the next free VMID
673 *
674 * @rdev: radeon_device pointer
675 * @vm: vm to allocate id for
676 * @ring: ring we want to submit job to
677 *
678 * Allocate an id for the vm (cayman+).
679 * Returns the fence we need to sync to (if any).
680 *
681 * Global and local mutex must be locked!
682 */
683struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
684 struct radeon_vm *vm, int ring)
685{
686 struct radeon_fence *best[RADEON_NUM_RINGS] = {};
687 unsigned choices[2] = {};
688 unsigned i;
689
690 /* check if the id is still valid */
691 if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id])
692 return NULL;
693
694 /* we definately need to flush */
695 radeon_fence_unref(&vm->last_flush);
696
697 /* skip over VMID 0, since it is the system VM */
698 for (i = 1; i < rdev->vm_manager.nvm; ++i) {
699 struct radeon_fence *fence = rdev->vm_manager.active[i];
700
701 if (fence == NULL) {
702 /* found a free one */
703 vm->id = i;
704 return NULL;
705 }
706
707 if (radeon_fence_is_earlier(fence, best[fence->ring])) {
708 best[fence->ring] = fence;
709 choices[fence->ring == ring ? 0 : 1] = i;
710 }
711 }
712
713 for (i = 0; i < 2; ++i) {
714 if (choices[i]) {
715 vm->id = choices[i];
716 return rdev->vm_manager.active[choices[i]];
717 }
718 }
719
720 /* should never happen */
721 BUG();
722 return NULL;
723}
724
725/**
726 * radeon_vm_fence - remember fence for vm
727 *
728 * @rdev: radeon_device pointer
729 * @vm: vm we want to fence
730 * @fence: fence to remember
731 *
732 * Fence the vm (cayman+).
733 * Set the fence used to protect page table and id.
734 *
735 * Global and local mutex must be locked!
736 */
737void radeon_vm_fence(struct radeon_device *rdev,
738 struct radeon_vm *vm,
739 struct radeon_fence *fence)
740{
741 radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
742 rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
743
744 radeon_fence_unref(&vm->fence);
745 vm->fence = radeon_fence_ref(fence);
746}
747
Christian König421ca7a2012-09-11 16:10:00 +0200748/**
749 * radeon_vm_bo_find - find the bo_va for a specific vm & bo
750 *
751 * @vm: requested vm
752 * @bo: requested buffer object
753 *
754 * Find @bo inside the requested vm (cayman+).
755 * Search inside the @bos vm list for the requested vm
756 * Returns the found bo_va or NULL if none is found
757 *
758 * Object has to be reserved!
759 */
760struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
761 struct radeon_bo *bo)
762{
763 struct radeon_bo_va *bo_va;
764
765 list_for_each_entry(bo_va, &bo->va, bo_list) {
766 if (bo_va->vm == vm) {
767 return bo_va;
768 }
769 }
770 return NULL;
771}
772
Alex Deucher09db8642012-07-17 14:02:40 -0400773/**
774 * radeon_vm_bo_add - add a bo to a specific vm
775 *
776 * @rdev: radeon_device pointer
777 * @vm: requested vm
778 * @bo: radeon buffer object
Alex Deucher09db8642012-07-17 14:02:40 -0400779 *
780 * Add @bo into the requested vm (cayman+).
Christian Könige971bd52012-09-11 16:10:04 +0200781 * Add @bo to the list of bos associated with the vm
782 * Returns newly added bo_va or NULL for failure
783 *
784 * Object has to be reserved!
Alex Deucher09db8642012-07-17 14:02:40 -0400785 */
Christian Könige971bd52012-09-11 16:10:04 +0200786struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
787 struct radeon_vm *vm,
788 struct radeon_bo *bo)
Jerome Glisse721604a2012-01-05 22:11:05 -0500789{
Christian Könige971bd52012-09-11 16:10:04 +0200790 struct radeon_bo_va *bo_va;
Jerome Glisse721604a2012-01-05 22:11:05 -0500791
792 bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL);
793 if (bo_va == NULL) {
Christian Könige971bd52012-09-11 16:10:04 +0200794 return NULL;
Jerome Glisse721604a2012-01-05 22:11:05 -0500795 }
796 bo_va->vm = vm;
797 bo_va->bo = bo;
Christian Könige971bd52012-09-11 16:10:04 +0200798 bo_va->soffset = 0;
799 bo_va->eoffset = 0;
800 bo_va->flags = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500801 bo_va->valid = false;
Christian Könige971bd52012-09-11 16:10:04 +0200802 bo_va->ref_count = 1;
Jerome Glisse721604a2012-01-05 22:11:05 -0500803 INIT_LIST_HEAD(&bo_va->bo_list);
804 INIT_LIST_HEAD(&bo_va->vm_list);
Jerome Glisse721604a2012-01-05 22:11:05 -0500805
Christian Könige971bd52012-09-11 16:10:04 +0200806 mutex_lock(&vm->mutex);
807 list_add(&bo_va->vm_list, &vm->va);
808 list_add_tail(&bo_va->bo_list, &bo->va);
809 mutex_unlock(&vm->mutex);
810
811 return bo_va;
812}
813
814/**
815 * radeon_vm_bo_set_addr - set bos virtual address inside a vm
816 *
817 * @rdev: radeon_device pointer
818 * @bo_va: bo_va to store the address
819 * @soffset: requested offset of the buffer in the VM address space
820 * @flags: attributes of pages (read/write/valid/etc.)
821 *
822 * Set offset of @bo_va (cayman+).
823 * Validate and set the offset requested within the vm address space.
Jerome Glisse721604a2012-01-05 22:11:05 -0500824 * Returns 0 for success, error for failure.
Christian König421ca7a2012-09-11 16:10:00 +0200825 *
826 * Object has to be reserved!
Jerome Glisse721604a2012-01-05 22:11:05 -0500827 */
Christian Könige971bd52012-09-11 16:10:04 +0200828int radeon_vm_bo_set_addr(struct radeon_device *rdev,
829 struct radeon_bo_va *bo_va,
830 uint64_t soffset,
831 uint32_t flags)
Jerome Glisse721604a2012-01-05 22:11:05 -0500832{
Christian Könige971bd52012-09-11 16:10:04 +0200833 uint64_t size = radeon_bo_size(bo_va->bo);
834 uint64_t eoffset, last_offset = 0;
835 struct radeon_vm *vm = bo_va->vm;
836 struct radeon_bo_va *tmp;
Jerome Glisse721604a2012-01-05 22:11:05 -0500837 struct list_head *head;
Jerome Glisse721604a2012-01-05 22:11:05 -0500838 unsigned last_pfn;
839
Christian Könige971bd52012-09-11 16:10:04 +0200840 if (soffset) {
841 /* make sure object fit at this offset */
842 eoffset = soffset + size;
843 if (soffset >= eoffset) {
844 return -EINVAL;
845 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500846
Christian Könige971bd52012-09-11 16:10:04 +0200847 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
848 if (last_pfn > rdev->vm_manager.max_pfn) {
849 dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
850 last_pfn, rdev->vm_manager.max_pfn);
851 return -EINVAL;
852 }
853
854 } else {
855 eoffset = last_pfn = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500856 }
857
858 mutex_lock(&vm->mutex);
Jerome Glisse721604a2012-01-05 22:11:05 -0500859 head = &vm->va;
860 last_offset = 0;
861 list_for_each_entry(tmp, &vm->va, vm_list) {
Christian Könige971bd52012-09-11 16:10:04 +0200862 if (bo_va == tmp) {
863 /* skip over currently modified bo */
864 continue;
865 }
866
867 if (soffset >= last_offset && eoffset <= tmp->soffset) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500868 /* bo can be added before this one */
869 break;
870 }
Christian Könige971bd52012-09-11 16:10:04 +0200871 if (eoffset > tmp->soffset && soffset < tmp->eoffset) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500872 /* bo and tmp overlap, invalid offset */
Jerome Glisse721604a2012-01-05 22:11:05 -0500873 dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n",
Christian Könige971bd52012-09-11 16:10:04 +0200874 bo_va->bo, (unsigned)bo_va->soffset, tmp->bo,
Jerome Glisse721604a2012-01-05 22:11:05 -0500875 (unsigned)tmp->soffset, (unsigned)tmp->eoffset);
876 mutex_unlock(&vm->mutex);
877 return -EINVAL;
878 }
879 last_offset = tmp->eoffset;
880 head = &tmp->vm_list;
881 }
Christian Könige971bd52012-09-11 16:10:04 +0200882
883 bo_va->soffset = soffset;
884 bo_va->eoffset = eoffset;
885 bo_va->flags = flags;
886 bo_va->valid = false;
887 list_move(&bo_va->vm_list, head);
888
Jerome Glisse721604a2012-01-05 22:11:05 -0500889 mutex_unlock(&vm->mutex);
890 return 0;
891}
892
Alex Deucher09db8642012-07-17 14:02:40 -0400893/**
Christian Königdce34bf2012-09-17 19:36:18 +0200894 * radeon_vm_map_gart - get the physical address of a gart page
Alex Deucher09db8642012-07-17 14:02:40 -0400895 *
896 * @rdev: radeon_device pointer
Christian Königdce34bf2012-09-17 19:36:18 +0200897 * @addr: the unmapped addr
Alex Deucher09db8642012-07-17 14:02:40 -0400898 *
899 * Look up the physical address of the page that the pte resolves
900 * to (cayman+).
901 * Returns the physical address of the page.
902 */
Christian Königdce34bf2012-09-17 19:36:18 +0200903uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
Jerome Glisse721604a2012-01-05 22:11:05 -0500904{
Christian Königdce34bf2012-09-17 19:36:18 +0200905 uint64_t result;
Jerome Glisse721604a2012-01-05 22:11:05 -0500906
Christian Königdce34bf2012-09-17 19:36:18 +0200907 /* page table offset */
908 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT];
909
910 /* in case cpu page size != gpu page size*/
911 result |= addr & (~PAGE_MASK);
912
913 return result;
Jerome Glisse721604a2012-01-05 22:11:05 -0500914}
915
Alex Deucher09db8642012-07-17 14:02:40 -0400916/**
Christian König24c16432013-10-30 11:51:09 -0400917 * radeon_vm_page_flags - translate page flags to what the hw uses
918 *
919 * @flags: flags comming from userspace
920 *
921 * Translate the flags the userspace ABI uses to hw flags.
922 */
923static uint32_t radeon_vm_page_flags(uint32_t flags)
924{
925 uint32_t hw_flags = 0;
926 hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
927 hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
928 hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
929 if (flags & RADEON_VM_PAGE_SYSTEM) {
930 hw_flags |= R600_PTE_SYSTEM;
931 hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
932 }
933 return hw_flags;
934}
935
936/**
Christian König90a51a32012-10-09 13:31:17 +0200937 * radeon_vm_update_pdes - make sure that page directory is valid
938 *
939 * @rdev: radeon_device pointer
940 * @vm: requested vm
941 * @start: start of GPU address range
942 * @end: end of GPU address range
943 *
944 * Allocates new page tables if necessary
945 * and updates the page directory (cayman+).
946 * Returns 0 for success, error for failure.
947 *
948 * Global and local mutex must be locked!
949 */
950static int radeon_vm_update_pdes(struct radeon_device *rdev,
951 struct radeon_vm *vm,
Alex Deucher43f12142013-02-01 17:32:42 +0100952 struct radeon_ib *ib,
Christian König90a51a32012-10-09 13:31:17 +0200953 uint64_t start, uint64_t end)
954{
955 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8;
956
957 uint64_t last_pde = ~0, last_pt = ~0;
958 unsigned count = 0;
959 uint64_t pt_idx;
960 int r;
961
962 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
963 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
964
965 /* walk over the address space and update the page directory */
966 for (pt_idx = start; pt_idx <= end; ++pt_idx) {
967 uint64_t pde, pt;
968
969 if (vm->page_tables[pt_idx])
970 continue;
971
972retry:
973 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
974 &vm->page_tables[pt_idx],
Alex Deucher3e3e53f2013-07-18 13:11:56 -0400975 RADEON_VM_PTE_COUNT * 8,
976 RADEON_GPU_PAGE_SIZE, false);
Christian König90a51a32012-10-09 13:31:17 +0200977
978 if (r == -ENOMEM) {
979 r = radeon_vm_evict(rdev, vm);
980 if (r)
981 return r;
982 goto retry;
983 } else if (r) {
984 return r;
985 }
986
987 pde = vm->pd_gpu_addr + pt_idx * 8;
988
989 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
990
991 if (((last_pde + 8 * count) != pde) ||
992 ((last_pt + incr * count) != pt)) {
993
994 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +0100995 radeon_asic_vm_set_page(rdev, ib, last_pde,
Christian König90a51a32012-10-09 13:31:17 +0200996 last_pt, count, incr,
Christian König24c16432013-10-30 11:51:09 -0400997 R600_PTE_VALID);
Christian König5b2906e2013-10-29 20:14:50 +0100998
999 count *= RADEON_VM_PTE_COUNT;
1000 radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
1001 count, 0, 0);
Christian König90a51a32012-10-09 13:31:17 +02001002 }
1003
1004 count = 1;
1005 last_pde = pde;
1006 last_pt = pt;
1007 } else {
1008 ++count;
1009 }
1010 }
1011
1012 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001013 radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count,
Christian König24c16432013-10-30 11:51:09 -04001014 incr, R600_PTE_VALID);
Christian König90a51a32012-10-09 13:31:17 +02001015
Christian König5b2906e2013-10-29 20:14:50 +01001016 count *= RADEON_VM_PTE_COUNT;
1017 radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
1018 count, 0, 0);
Christian König90a51a32012-10-09 13:31:17 +02001019 }
1020
1021 return 0;
1022}
1023
1024/**
1025 * radeon_vm_update_ptes - make sure that page tables are valid
1026 *
1027 * @rdev: radeon_device pointer
1028 * @vm: requested vm
1029 * @start: start of GPU address range
1030 * @end: end of GPU address range
1031 * @dst: destination address to map to
1032 * @flags: mapping flags
1033 *
1034 * Update the page tables in the range @start - @end (cayman+).
1035 *
1036 * Global and local mutex must be locked!
1037 */
1038static void radeon_vm_update_ptes(struct radeon_device *rdev,
1039 struct radeon_vm *vm,
Alex Deucher43f12142013-02-01 17:32:42 +01001040 struct radeon_ib *ib,
Christian König90a51a32012-10-09 13:31:17 +02001041 uint64_t start, uint64_t end,
1042 uint64_t dst, uint32_t flags)
1043{
1044 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1;
1045
1046 uint64_t last_pte = ~0, last_dst = ~0;
1047 unsigned count = 0;
1048 uint64_t addr;
1049
1050 start = start / RADEON_GPU_PAGE_SIZE;
1051 end = end / RADEON_GPU_PAGE_SIZE;
1052
1053 /* walk over the address space and update the page tables */
1054 for (addr = start; addr < end; ) {
1055 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE;
1056 unsigned nptes;
1057 uint64_t pte;
1058
1059 if ((addr & ~mask) == (end & ~mask))
1060 nptes = end - addr;
1061 else
1062 nptes = RADEON_VM_PTE_COUNT - (addr & mask);
1063
1064 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
1065 pte += (addr & mask) * 8;
1066
Christian König204a3932012-10-22 17:42:38 +02001067 if ((last_pte + 8 * count) != pte) {
Christian König90a51a32012-10-09 13:31:17 +02001068
1069 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001070 radeon_asic_vm_set_page(rdev, ib, last_pte,
Christian König90a51a32012-10-09 13:31:17 +02001071 last_dst, count,
1072 RADEON_GPU_PAGE_SIZE,
1073 flags);
1074 }
1075
1076 count = nptes;
1077 last_pte = pte;
1078 last_dst = dst;
1079 } else {
1080 count += nptes;
1081 }
1082
1083 addr += nptes;
1084 dst += nptes * RADEON_GPU_PAGE_SIZE;
1085 }
1086
1087 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001088 radeon_asic_vm_set_page(rdev, ib, last_pte,
1089 last_dst, count,
Christian König90a51a32012-10-09 13:31:17 +02001090 RADEON_GPU_PAGE_SIZE, flags);
1091 }
1092}
1093
1094/**
Alex Deucher09db8642012-07-17 14:02:40 -04001095 * radeon_vm_bo_update_pte - map a bo into the vm page table
1096 *
1097 * @rdev: radeon_device pointer
1098 * @vm: requested vm
1099 * @bo: radeon buffer object
1100 * @mem: ttm mem
1101 *
1102 * Fill in the page table entries for @bo (cayman+).
1103 * Returns 0 for success, -EINVAL for failure.
Christian König2a6f1ab2012-08-11 15:00:30 +02001104 *
1105 * Object have to be reserved & global and local mutex must be locked!
Alex Deucher09db8642012-07-17 14:02:40 -04001106 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001107int radeon_vm_bo_update_pte(struct radeon_device *rdev,
1108 struct radeon_vm *vm,
1109 struct radeon_bo *bo,
1110 struct ttm_mem_reg *mem)
1111{
Alex Deucher43f12142013-02-01 17:32:42 +01001112 struct radeon_ib ib;
Jerome Glisse721604a2012-01-05 22:11:05 -05001113 struct radeon_bo_va *bo_va;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001114 unsigned nptes, npdes, ndw;
Christian König90a51a32012-10-09 13:31:17 +02001115 uint64_t addr;
Christian König2a6f1ab2012-08-11 15:00:30 +02001116 int r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001117
1118 /* nothing to do if vm isn't bound */
Christian König90a51a32012-10-09 13:31:17 +02001119 if (vm->page_directory == NULL)
Jesper Juhl04bd27a2012-02-26 23:51:53 +01001120 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001121
Christian König421ca7a2012-09-11 16:10:00 +02001122 bo_va = radeon_vm_bo_find(vm, bo);
Jerome Glisse721604a2012-01-05 22:11:05 -05001123 if (bo_va == NULL) {
1124 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
1125 return -EINVAL;
1126 }
1127
Christian Könige971bd52012-09-11 16:10:04 +02001128 if (!bo_va->soffset) {
1129 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
1130 bo, vm);
1131 return -EINVAL;
1132 }
1133
Christian König2a6f1ab2012-08-11 15:00:30 +02001134 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
Jerome Glisse721604a2012-01-05 22:11:05 -05001135 return 0;
1136
Jerome Glisse721604a2012-01-05 22:11:05 -05001137 bo_va->flags &= ~RADEON_VM_PAGE_VALID;
1138 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
1139 if (mem) {
Christian Königdce34bf2012-09-17 19:36:18 +02001140 addr = mem->start << PAGE_SHIFT;
Jerome Glisse721604a2012-01-05 22:11:05 -05001141 if (mem->mem_type != TTM_PL_SYSTEM) {
1142 bo_va->flags |= RADEON_VM_PAGE_VALID;
1143 bo_va->valid = true;
1144 }
1145 if (mem->mem_type == TTM_PL_TT) {
1146 bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
Christian Königdce34bf2012-09-17 19:36:18 +02001147 } else {
1148 addr += rdev->vm_manager.vram_base_offset;
Christian König2a6f1ab2012-08-11 15:00:30 +02001149 }
1150 } else {
Christian Königdce34bf2012-09-17 19:36:18 +02001151 addr = 0;
Christian König2a6f1ab2012-08-11 15:00:30 +02001152 bo_va->valid = false;
Jerome Glisse721604a2012-01-05 22:11:05 -05001153 }
Christian König2a6f1ab2012-08-11 15:00:30 +02001154
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001155 nptes = radeon_bo_ngpu_pages(bo);
1156
Christian König90a51a32012-10-09 13:31:17 +02001157 /* assume two extra pdes in case the mapping overlaps the borders */
1158 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001159
Alex Deucher43f12142013-02-01 17:32:42 +01001160 /* padding, etc. */
1161 ndw = 64;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001162
Christian König90a51a32012-10-09 13:31:17 +02001163 if (RADEON_VM_BLOCK_SIZE > 11)
1164 /* reserve space for one header for every 2k dwords */
Christian König08eda322012-10-22 17:42:39 +02001165 ndw += (nptes >> 11) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001166 else
1167 /* reserve space for one header for
1168 every (1 << BLOCK_SIZE) entries */
Christian König08eda322012-10-22 17:42:39 +02001169 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001170
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001171 /* reserve space for pte addresses */
1172 ndw += nptes * 2;
1173
1174 /* reserve space for one header for every 2k dwords */
Christian König08eda322012-10-22 17:42:39 +02001175 ndw += (npdes >> 11) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001176
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001177 /* reserve space for pde addresses */
1178 ndw += npdes * 2;
Christian König2a6f1ab2012-08-11 15:00:30 +02001179
Christian König5b2906e2013-10-29 20:14:50 +01001180 /* reserve space for clearing new page tables */
1181 ndw += npdes * 2 * RADEON_VM_PTE_COUNT;
1182
Alex Deucher43f12142013-02-01 17:32:42 +01001183 /* update too big for an IB */
1184 if (ndw > 0xfffff)
1185 return -ENOMEM;
1186
Christian König24c16432013-10-30 11:51:09 -04001187 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
Alex Deucher43f12142013-02-01 17:32:42 +01001188 ib.length_dw = 0;
1189
1190 r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset);
Christian König2a6f1ab2012-08-11 15:00:30 +02001191 if (r) {
Alex Deucher43f12142013-02-01 17:32:42 +01001192 radeon_ib_free(rdev, &ib);
Christian König2a6f1ab2012-08-11 15:00:30 +02001193 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001194 }
Christian König2a6f1ab2012-08-11 15:00:30 +02001195
Alex Deucher43f12142013-02-01 17:32:42 +01001196 radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset,
Christian König24c16432013-10-30 11:51:09 -04001197 addr, radeon_vm_page_flags(bo_va->flags));
Christian König2a6f1ab2012-08-11 15:00:30 +02001198
Alex Deucher43f12142013-02-01 17:32:42 +01001199 radeon_ib_sync_to(&ib, vm->fence);
1200 r = radeon_ib_schedule(rdev, &ib, NULL);
Christian König2a6f1ab2012-08-11 15:00:30 +02001201 if (r) {
Alex Deucher43f12142013-02-01 17:32:42 +01001202 radeon_ib_free(rdev, &ib);
Christian König2a6f1ab2012-08-11 15:00:30 +02001203 return r;
1204 }
Alex Deucher43f12142013-02-01 17:32:42 +01001205 radeon_fence_unref(&vm->fence);
1206 vm->fence = radeon_fence_ref(ib.fence);
1207 radeon_ib_free(rdev, &ib);
Christian König9b40e5d2012-08-08 12:22:43 +02001208 radeon_fence_unref(&vm->last_flush);
Christian König90a51a32012-10-09 13:31:17 +02001209
Jerome Glisse721604a2012-01-05 22:11:05 -05001210 return 0;
1211}
1212
Alex Deucher09db8642012-07-17 14:02:40 -04001213/**
1214 * radeon_vm_bo_rmv - remove a bo to a specific vm
1215 *
1216 * @rdev: radeon_device pointer
Christian Könige971bd52012-09-11 16:10:04 +02001217 * @bo_va: requested bo_va
Alex Deucher09db8642012-07-17 14:02:40 -04001218 *
Christian Könige971bd52012-09-11 16:10:04 +02001219 * Remove @bo_va->bo from the requested vm (cayman+).
1220 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and
1221 * remove the ptes for @bo_va in the page table.
Alex Deucher09db8642012-07-17 14:02:40 -04001222 * Returns 0 for success.
Christian Königddf03f52012-08-09 20:02:28 +02001223 *
1224 * Object have to be reserved!
Alex Deucher09db8642012-07-17 14:02:40 -04001225 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001226int radeon_vm_bo_rmv(struct radeon_device *rdev,
Christian Könige971bd52012-09-11 16:10:04 +02001227 struct radeon_bo_va *bo_va)
Jerome Glisse721604a2012-01-05 22:11:05 -05001228{
Jerome Glisse3813f5c2013-06-06 12:41:17 -04001229 int r = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001230
Christian König36ff39c2012-05-09 10:07:08 +02001231 mutex_lock(&rdev->vm_manager.lock);
Christian Könige971bd52012-09-11 16:10:04 +02001232 mutex_lock(&bo_va->vm->mutex);
Jerome Glisse3813f5c2013-06-06 12:41:17 -04001233 if (bo_va->soffset) {
1234 r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL);
1235 }
Christian König36ff39c2012-05-09 10:07:08 +02001236 mutex_unlock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -05001237 list_del(&bo_va->vm_list);
Christian Könige971bd52012-09-11 16:10:04 +02001238 mutex_unlock(&bo_va->vm->mutex);
Sebastian Biemueller108b0d32012-02-29 11:04:52 -05001239 list_del(&bo_va->bo_list);
Jerome Glisse721604a2012-01-05 22:11:05 -05001240
1241 kfree(bo_va);
Christian König2a6f1ab2012-08-11 15:00:30 +02001242 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001243}
1244
Alex Deucher09db8642012-07-17 14:02:40 -04001245/**
1246 * radeon_vm_bo_invalidate - mark the bo as invalid
1247 *
1248 * @rdev: radeon_device pointer
1249 * @vm: requested vm
1250 * @bo: radeon buffer object
1251 *
1252 * Mark @bo as invalid (cayman+).
1253 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001254void radeon_vm_bo_invalidate(struct radeon_device *rdev,
1255 struct radeon_bo *bo)
1256{
1257 struct radeon_bo_va *bo_va;
1258
Jerome Glisse721604a2012-01-05 22:11:05 -05001259 list_for_each_entry(bo_va, &bo->va, bo_list) {
1260 bo_va->valid = false;
1261 }
1262}
1263
Alex Deucher09db8642012-07-17 14:02:40 -04001264/**
1265 * radeon_vm_init - initialize a vm instance
1266 *
1267 * @rdev: radeon_device pointer
1268 * @vm: requested vm
1269 *
Christian Königd72d43c2012-10-09 13:31:18 +02001270 * Init @vm fields (cayman+).
Alex Deucher09db8642012-07-17 14:02:40 -04001271 */
Christian Königd72d43c2012-10-09 13:31:18 +02001272void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
Jerome Glisse721604a2012-01-05 22:11:05 -05001273{
Christian Königee60e292012-08-09 16:21:08 +02001274 vm->id = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001275 vm->fence = NULL;
1276 mutex_init(&vm->mutex);
1277 INIT_LIST_HEAD(&vm->list);
1278 INIT_LIST_HEAD(&vm->va);
Jerome Glisse721604a2012-01-05 22:11:05 -05001279}
1280
Alex Deucher09db8642012-07-17 14:02:40 -04001281/**
Dmitrii Cherkasovf59abbf2012-08-13 10:53:29 -04001282 * radeon_vm_fini - tear down a vm instance
Alex Deucher09db8642012-07-17 14:02:40 -04001283 *
1284 * @rdev: radeon_device pointer
1285 * @vm: requested vm
1286 *
1287 * Tear down @vm (cayman+).
1288 * Unbind the VM and remove all bos from the vm bo list
1289 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001290void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
1291{
1292 struct radeon_bo_va *bo_va, *tmp;
1293 int r;
1294
Christian König36ff39c2012-05-09 10:07:08 +02001295 mutex_lock(&rdev->vm_manager.lock);
Christian Königbb409152012-06-03 16:09:43 +02001296 mutex_lock(&vm->mutex);
Christian Königddf03f52012-08-09 20:02:28 +02001297 radeon_vm_free_pt(rdev, vm);
Christian König36ff39c2012-05-09 10:07:08 +02001298 mutex_unlock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -05001299
Jerome Glisse721604a2012-01-05 22:11:05 -05001300 if (!list_empty(&vm->va)) {
1301 dev_err(rdev->dev, "still active bo inside vm\n");
1302 }
1303 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) {
1304 list_del_init(&bo_va->vm_list);
1305 r = radeon_bo_reserve(bo_va->bo, false);
1306 if (!r) {
1307 list_del_init(&bo_va->bo_list);
1308 radeon_bo_unreserve(bo_va->bo);
1309 kfree(bo_va);
1310 }
1311 }
Christian Königddf03f52012-08-09 20:02:28 +02001312 radeon_fence_unref(&vm->fence);
1313 radeon_fence_unref(&vm->last_flush);
Jerome Glisse721604a2012-01-05 22:11:05 -05001314 mutex_unlock(&vm->mutex);
1315}