blob: 84c53e41a88fd783d6f7ba32c9155109eb3c95be [file] [log] [blame]
Jerome Glisse771fe6b2009-06-05 14:42:42 +02001/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32#include <ttm/ttm_bo_api.h>
33#include <ttm/ttm_bo_driver.h>
34#include <ttm/ttm_placement.h>
35#include <ttm/ttm_module.h>
Pauli Nieminen8d7cddc2010-04-01 12:44:59 +000036#include <ttm/ttm_page_alloc.h>
Jerome Glisse771fe6b2009-06-05 14:42:42 +020037#include <drm/drmP.h>
38#include <drm/radeon_drm.h>
Dave Airliefa8a1232009-08-26 13:13:37 +100039#include <linux/seq_file.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090040#include <linux/slab.h>
Jerome Glisse771fe6b2009-06-05 14:42:42 +020041#include "radeon_reg.h"
42#include "radeon.h"
43
44#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
45
Dave Airliefa8a1232009-08-26 13:13:37 +100046static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
47
Jerome Glisse771fe6b2009-06-05 14:42:42 +020048static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev)
49{
50 struct radeon_mman *mman;
51 struct radeon_device *rdev;
52
53 mman = container_of(bdev, struct radeon_mman, bdev);
54 rdev = container_of(mman, struct radeon_device, mman);
55 return rdev;
56}
57
58
59/*
60 * Global memory.
61 */
Dave Airlieba4420c2010-03-09 10:56:52 +100062static int radeon_ttm_mem_global_init(struct drm_global_reference *ref)
Jerome Glisse771fe6b2009-06-05 14:42:42 +020063{
64 return ttm_mem_global_init(ref->object);
65}
66
Dave Airlieba4420c2010-03-09 10:56:52 +100067static void radeon_ttm_mem_global_release(struct drm_global_reference *ref)
Jerome Glisse771fe6b2009-06-05 14:42:42 +020068{
69 ttm_mem_global_release(ref->object);
70}
71
72static int radeon_ttm_global_init(struct radeon_device *rdev)
73{
Dave Airlieba4420c2010-03-09 10:56:52 +100074 struct drm_global_reference *global_ref;
Jerome Glisse771fe6b2009-06-05 14:42:42 +020075 int r;
76
77 rdev->mman.mem_global_referenced = false;
78 global_ref = &rdev->mman.mem_global_ref;
Dave Airlieba4420c2010-03-09 10:56:52 +100079 global_ref->global_type = DRM_GLOBAL_TTM_MEM;
Jerome Glisse771fe6b2009-06-05 14:42:42 +020080 global_ref->size = sizeof(struct ttm_mem_global);
81 global_ref->init = &radeon_ttm_mem_global_init;
82 global_ref->release = &radeon_ttm_mem_global_release;
Dave Airlieba4420c2010-03-09 10:56:52 +100083 r = drm_global_item_ref(global_ref);
Jerome Glisse771fe6b2009-06-05 14:42:42 +020084 if (r != 0) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +020085 DRM_ERROR("Failed setting up TTM memory accounting "
86 "subsystem.\n");
Jerome Glisse771fe6b2009-06-05 14:42:42 +020087 return r;
88 }
Thomas Hellstroma987fca2009-08-18 16:51:56 +020089
90 rdev->mman.bo_global_ref.mem_glob =
91 rdev->mman.mem_global_ref.object;
92 global_ref = &rdev->mman.bo_global_ref.ref;
Dave Airlieba4420c2010-03-09 10:56:52 +100093 global_ref->global_type = DRM_GLOBAL_TTM_BO;
Thomas Hellstrom7f5f4db2009-08-20 10:29:08 +020094 global_ref->size = sizeof(struct ttm_bo_global);
Thomas Hellstroma987fca2009-08-18 16:51:56 +020095 global_ref->init = &ttm_bo_global_init;
96 global_ref->release = &ttm_bo_global_release;
Dave Airlieba4420c2010-03-09 10:56:52 +100097 r = drm_global_item_ref(global_ref);
Thomas Hellstroma987fca2009-08-18 16:51:56 +020098 if (r != 0) {
99 DRM_ERROR("Failed setting up TTM BO subsystem.\n");
Dave Airlieba4420c2010-03-09 10:56:52 +1000100 drm_global_item_unref(&rdev->mman.mem_global_ref);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200101 return r;
102 }
103
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200104 rdev->mman.mem_global_referenced = true;
105 return 0;
106}
107
108static void radeon_ttm_global_fini(struct radeon_device *rdev)
109{
110 if (rdev->mman.mem_global_referenced) {
Dave Airlieba4420c2010-03-09 10:56:52 +1000111 drm_global_item_unref(&rdev->mman.bo_global_ref.ref);
112 drm_global_item_unref(&rdev->mman.mem_global_ref);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200113 rdev->mman.mem_global_referenced = false;
114 }
115}
116
117struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev);
118
119static struct ttm_backend*
120radeon_create_ttm_backend_entry(struct ttm_bo_device *bdev)
121{
122 struct radeon_device *rdev;
123
124 rdev = radeon_get_rdev(bdev);
125#if __OS_HAS_AGP
126 if (rdev->flags & RADEON_IS_AGP) {
127 return ttm_agp_backend_init(bdev, rdev->ddev->agp->bridge);
128 } else
129#endif
130 {
131 return radeon_ttm_backend_create(rdev);
132 }
133}
134
135static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
136{
137 return 0;
138}
139
140static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
141 struct ttm_mem_type_manager *man)
142{
143 struct radeon_device *rdev;
144
145 rdev = radeon_get_rdev(bdev);
146
147 switch (type) {
148 case TTM_PL_SYSTEM:
149 /* System memory */
150 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
151 man->available_caching = TTM_PL_MASK_CACHING;
152 man->default_caching = TTM_PL_FLAG_CACHED;
153 break;
154 case TTM_PL_TT:
Jerome Glissed594e462010-02-17 21:54:29 +0000155 man->gpu_offset = rdev->mc.gtt_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200156 man->available_caching = TTM_PL_MASK_CACHING;
157 man->default_caching = TTM_PL_FLAG_CACHED;
Michel Dänzer55c93272009-06-15 16:56:11 +0200158 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200159#if __OS_HAS_AGP
160 if (rdev->flags & RADEON_IS_AGP) {
161 if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) {
162 DRM_ERROR("AGP is not enabled for memory type %u\n",
163 (unsigned)type);
164 return -EINVAL;
165 }
Michel Dänzer55c93272009-06-15 16:56:11 +0200166 if (!rdev->ddev->agp->cant_use_aperture)
Jerome Glisse0a2d50e2010-04-09 14:39:24 +0200167 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200168 man->available_caching = TTM_PL_FLAG_UNCACHED |
169 TTM_PL_FLAG_WC;
170 man->default_caching = TTM_PL_FLAG_WC;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200171 }
Jerome Glisse0c321c72010-04-07 10:21:27 +0000172#endif
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200173 break;
174 case TTM_PL_VRAM:
175 /* "On-card" video ram */
Jerome Glissed594e462010-02-17 21:54:29 +0000176 man->gpu_offset = rdev->mc.vram_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200177 man->flags = TTM_MEMTYPE_FLAG_FIXED |
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200178 TTM_MEMTYPE_FLAG_MAPPABLE;
179 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
180 man->default_caching = TTM_PL_FLAG_WC;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200181 break;
182 default:
183 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
184 return -EINVAL;
185 }
186 return 0;
187}
188
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100189static void radeon_evict_flags(struct ttm_buffer_object *bo,
190 struct ttm_placement *placement)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200191{
Jerome Glissed03d8582009-12-14 21:02:09 +0100192 struct radeon_bo *rbo;
193 static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
194
195 if (!radeon_ttm_bo_is_radeon_bo(bo)) {
196 placement->fpfn = 0;
197 placement->lpfn = 0;
198 placement->placement = &placements;
199 placement->busy_placement = &placements;
200 placement->num_placement = 1;
201 placement->num_busy_placement = 1;
202 return;
203 }
204 rbo = container_of(bo, struct radeon_bo, tbo);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200205 switch (bo->mem.mem_type) {
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100206 case TTM_PL_VRAM:
Dave Airlie9270eb12010-01-13 09:21:49 +1000207 if (rbo->rdev->cp.ready == false)
208 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
209 else
210 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100211 break;
212 case TTM_PL_TT:
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200213 default:
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100214 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200215 }
Jerome Glisseeaa5fd12009-12-09 21:57:37 +0100216 *placement = rbo->placement;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200217}
218
219static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp)
220{
221 return 0;
222}
223
224static void radeon_move_null(struct ttm_buffer_object *bo,
225 struct ttm_mem_reg *new_mem)
226{
227 struct ttm_mem_reg *old_mem = &bo->mem;
228
229 BUG_ON(old_mem->mm_node != NULL);
230 *old_mem = *new_mem;
231 new_mem->mm_node = NULL;
232}
233
234static int radeon_move_blit(struct ttm_buffer_object *bo,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000235 bool evict, int no_wait_reserve, bool no_wait_gpu,
236 struct ttm_mem_reg *new_mem,
237 struct ttm_mem_reg *old_mem)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200238{
239 struct radeon_device *rdev;
240 uint64_t old_start, new_start;
241 struct radeon_fence *fence;
242 int r;
243
244 rdev = radeon_get_rdev(bo->bdev);
245 r = radeon_fence_create(rdev, &fence);
246 if (unlikely(r)) {
247 return r;
248 }
249 old_start = old_mem->mm_node->start << PAGE_SHIFT;
250 new_start = new_mem->mm_node->start << PAGE_SHIFT;
251
252 switch (old_mem->mem_type) {
253 case TTM_PL_VRAM:
Jerome Glissed594e462010-02-17 21:54:29 +0000254 old_start += rdev->mc.vram_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200255 break;
256 case TTM_PL_TT:
Jerome Glissed594e462010-02-17 21:54:29 +0000257 old_start += rdev->mc.gtt_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200258 break;
259 default:
260 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
261 return -EINVAL;
262 }
263 switch (new_mem->mem_type) {
264 case TTM_PL_VRAM:
Jerome Glissed594e462010-02-17 21:54:29 +0000265 new_start += rdev->mc.vram_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200266 break;
267 case TTM_PL_TT:
Jerome Glissed594e462010-02-17 21:54:29 +0000268 new_start += rdev->mc.gtt_start;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200269 break;
270 default:
271 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
272 return -EINVAL;
273 }
274 if (!rdev->cp.ready) {
275 DRM_ERROR("Trying to move memory with CP turned off.\n");
276 return -EINVAL;
277 }
278 r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
279 /* FIXME: handle copy error */
280 r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000281 evict, no_wait_reserve, no_wait_gpu, new_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200282 radeon_fence_unref(&fence);
283 return r;
284}
285
286static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000287 bool evict, bool interruptible,
288 bool no_wait_reserve, bool no_wait_gpu,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200289 struct ttm_mem_reg *new_mem)
290{
291 struct radeon_device *rdev;
292 struct ttm_mem_reg *old_mem = &bo->mem;
293 struct ttm_mem_reg tmp_mem;
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100294 u32 placements;
295 struct ttm_placement placement;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200296 int r;
297
298 rdev = radeon_get_rdev(bo->bdev);
299 tmp_mem = *new_mem;
300 tmp_mem.mm_node = NULL;
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100301 placement.fpfn = 0;
302 placement.lpfn = 0;
303 placement.num_placement = 1;
304 placement.placement = &placements;
305 placement.num_busy_placement = 1;
306 placement.busy_placement = &placements;
307 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
308 r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000309 interruptible, no_wait_reserve, no_wait_gpu);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200310 if (unlikely(r)) {
311 return r;
312 }
Dave Airliedf67bed2009-10-30 13:31:26 +1000313
314 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
315 if (unlikely(r)) {
316 goto out_cleanup;
317 }
318
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200319 r = ttm_tt_bind(bo->ttm, &tmp_mem);
320 if (unlikely(r)) {
321 goto out_cleanup;
322 }
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000323 r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem, old_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200324 if (unlikely(r)) {
325 goto out_cleanup;
326 }
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000327 r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200328out_cleanup:
329 if (tmp_mem.mm_node) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200330 struct ttm_bo_global *glob = rdev->mman.bdev.glob;
331
332 spin_lock(&glob->lru_lock);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200333 drm_mm_put_block(tmp_mem.mm_node);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200334 spin_unlock(&glob->lru_lock);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200335 return r;
336 }
337 return r;
338}
339
340static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000341 bool evict, bool interruptible,
342 bool no_wait_reserve, bool no_wait_gpu,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200343 struct ttm_mem_reg *new_mem)
344{
345 struct radeon_device *rdev;
346 struct ttm_mem_reg *old_mem = &bo->mem;
347 struct ttm_mem_reg tmp_mem;
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100348 struct ttm_placement placement;
349 u32 placements;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200350 int r;
351
352 rdev = radeon_get_rdev(bo->bdev);
353 tmp_mem = *new_mem;
354 tmp_mem.mm_node = NULL;
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100355 placement.fpfn = 0;
356 placement.lpfn = 0;
357 placement.num_placement = 1;
358 placement.placement = &placements;
359 placement.num_busy_placement = 1;
360 placement.busy_placement = &placements;
361 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000362 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_reserve, no_wait_gpu);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200363 if (unlikely(r)) {
364 return r;
365 }
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000366 r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200367 if (unlikely(r)) {
368 goto out_cleanup;
369 }
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000370 r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, new_mem, old_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200371 if (unlikely(r)) {
372 goto out_cleanup;
373 }
374out_cleanup:
375 if (tmp_mem.mm_node) {
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200376 struct ttm_bo_global *glob = rdev->mman.bdev.glob;
377
378 spin_lock(&glob->lru_lock);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200379 drm_mm_put_block(tmp_mem.mm_node);
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200380 spin_unlock(&glob->lru_lock);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200381 return r;
382 }
383 return r;
384}
385
386static int radeon_bo_move(struct ttm_buffer_object *bo,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000387 bool evict, bool interruptible,
388 bool no_wait_reserve, bool no_wait_gpu,
389 struct ttm_mem_reg *new_mem)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200390{
391 struct radeon_device *rdev;
392 struct ttm_mem_reg *old_mem = &bo->mem;
393 int r;
394
395 rdev = radeon_get_rdev(bo->bdev);
396 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
397 radeon_move_null(bo, new_mem);
398 return 0;
399 }
400 if ((old_mem->mem_type == TTM_PL_TT &&
401 new_mem->mem_type == TTM_PL_SYSTEM) ||
402 (old_mem->mem_type == TTM_PL_SYSTEM &&
403 new_mem->mem_type == TTM_PL_TT)) {
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200404 /* bind is enough */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200405 radeon_move_null(bo, new_mem);
406 return 0;
407 }
Jerome Glisse3ce0a232009-09-08 10:10:24 +1000408 if (!rdev->cp.ready || rdev->asic->copy == NULL) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200409 /* use memcpy */
Michel Dänzer1ab2e102009-07-28 12:30:56 +0200410 goto memcpy;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200411 }
412
413 if (old_mem->mem_type == TTM_PL_VRAM &&
414 new_mem->mem_type == TTM_PL_SYSTEM) {
Michel Dänzer1ab2e102009-07-28 12:30:56 +0200415 r = radeon_move_vram_ram(bo, evict, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000416 no_wait_reserve, no_wait_gpu, new_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200417 } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
418 new_mem->mem_type == TTM_PL_VRAM) {
Michel Dänzer1ab2e102009-07-28 12:30:56 +0200419 r = radeon_move_ram_vram(bo, evict, interruptible,
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000420 no_wait_reserve, no_wait_gpu, new_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200421 } else {
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000422 r = radeon_move_blit(bo, evict, no_wait_reserve, no_wait_gpu, new_mem, old_mem);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200423 }
Michel Dänzer1ab2e102009-07-28 12:30:56 +0200424
425 if (r) {
426memcpy:
Jerome Glisse9d87fa22010-04-07 10:21:19 +0000427 r = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
Michel Dänzer1ab2e102009-07-28 12:30:56 +0200428 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200429 return r;
430}
431
Jerome Glisse0a2d50e2010-04-09 14:39:24 +0200432static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
433{
434 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
435 struct radeon_device *rdev = radeon_get_rdev(bdev);
436
437 mem->bus.addr = NULL;
438 mem->bus.offset = 0;
439 mem->bus.size = mem->num_pages << PAGE_SHIFT;
440 mem->bus.base = 0;
441 mem->bus.is_iomem = false;
442 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
443 return -EINVAL;
444 switch (mem->mem_type) {
445 case TTM_PL_SYSTEM:
446 /* system memory */
447 return 0;
448 case TTM_PL_TT:
449#if __OS_HAS_AGP
450 if (rdev->flags & RADEON_IS_AGP) {
451 /* RADEON_IS_AGP is set only if AGP is active */
452 mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
453 mem->bus.base = rdev->mc.agp_base;
Michel Dänzer365048f2010-05-19 12:46:22 +0200454 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
Jerome Glisse0a2d50e2010-04-09 14:39:24 +0200455 }
456#endif
457 break;
458 case TTM_PL_VRAM:
459 mem->bus.offset = mem->mm_node->start << PAGE_SHIFT;
460 /* check if it's visible */
461 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
462 return -EINVAL;
463 mem->bus.base = rdev->mc.aper_base;
464 mem->bus.is_iomem = true;
465 break;
466 default:
467 return -EINVAL;
468 }
469 return 0;
470}
471
472static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
473{
474}
475
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200476static int radeon_sync_obj_wait(void *sync_obj, void *sync_arg,
477 bool lazy, bool interruptible)
478{
479 return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible);
480}
481
482static int radeon_sync_obj_flush(void *sync_obj, void *sync_arg)
483{
484 return 0;
485}
486
487static void radeon_sync_obj_unref(void **sync_obj)
488{
489 radeon_fence_unref((struct radeon_fence **)sync_obj);
490}
491
492static void *radeon_sync_obj_ref(void *sync_obj)
493{
494 return radeon_fence_ref((struct radeon_fence *)sync_obj);
495}
496
497static bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg)
498{
499 return radeon_fence_signaled((struct radeon_fence *)sync_obj);
500}
501
502static struct ttm_bo_driver radeon_bo_driver = {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200503 .create_ttm_backend_entry = &radeon_create_ttm_backend_entry,
504 .invalidate_caches = &radeon_invalidate_caches,
505 .init_mem_type = &radeon_init_mem_type,
506 .evict_flags = &radeon_evict_flags,
507 .move = &radeon_bo_move,
508 .verify_access = &radeon_verify_access,
509 .sync_obj_signaled = &radeon_sync_obj_signaled,
510 .sync_obj_wait = &radeon_sync_obj_wait,
511 .sync_obj_flush = &radeon_sync_obj_flush,
512 .sync_obj_unref = &radeon_sync_obj_unref,
513 .sync_obj_ref = &radeon_sync_obj_ref,
Dave Airliee024e112009-06-24 09:48:08 +1000514 .move_notify = &radeon_bo_move_notify,
515 .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
Jerome Glisse0a2d50e2010-04-09 14:39:24 +0200516 .io_mem_reserve = &radeon_ttm_io_mem_reserve,
517 .io_mem_free = &radeon_ttm_io_mem_free,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200518};
519
520int radeon_ttm_init(struct radeon_device *rdev)
521{
522 int r;
523
524 r = radeon_ttm_global_init(rdev);
525 if (r) {
526 return r;
527 }
528 /* No others user of address space so set it to 0 */
529 r = ttm_bo_device_init(&rdev->mman.bdev,
Thomas Hellstroma987fca2009-08-18 16:51:56 +0200530 rdev->mman.bo_global_ref.ref.object,
Dave Airliead49f502009-07-10 22:36:26 +1000531 &radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
532 rdev->need_dma32);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200533 if (r) {
534 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
535 return r;
536 }
Jerome Glisse0a0c7592009-12-11 20:36:19 +0100537 rdev->mman.initialized = true;
Jerome Glisse4c788672009-11-20 14:29:23 +0100538 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM,
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100539 rdev->mc.real_vram_size >> PAGE_SHIFT);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200540 if (r) {
541 DRM_ERROR("Failed initializing VRAM heap.\n");
542 return r;
543 }
Jerome Glisse4c788672009-11-20 14:29:23 +0100544 r = radeon_bo_create(rdev, NULL, 256 * 1024, true,
545 RADEON_GEM_DOMAIN_VRAM,
546 &rdev->stollen_vga_memory);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200547 if (r) {
548 return r;
549 }
Jerome Glisse4c788672009-11-20 14:29:23 +0100550 r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
551 if (r)
552 return r;
553 r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
554 radeon_bo_unreserve(rdev->stollen_vga_memory);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200555 if (r) {
Jerome Glisse4c788672009-11-20 14:29:23 +0100556 radeon_bo_unref(&rdev->stollen_vga_memory);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200557 return r;
558 }
559 DRM_INFO("radeon: %uM of VRAM memory ready\n",
Jerome Glisse3ce0a232009-09-08 10:10:24 +1000560 (unsigned)rdev->mc.real_vram_size / (1024 * 1024));
Jerome Glisse4c788672009-11-20 14:29:23 +0100561 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT,
Jerome Glisse312ea8d2009-12-07 15:52:58 +0100562 rdev->mc.gtt_size >> PAGE_SHIFT);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200563 if (r) {
564 DRM_ERROR("Failed initializing GTT heap.\n");
565 return r;
566 }
567 DRM_INFO("radeon: %uM of GTT memory ready.\n",
Jerome Glisse3ce0a232009-09-08 10:10:24 +1000568 (unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200569 if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
570 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
571 }
Dave Airliefa8a1232009-08-26 13:13:37 +1000572
573 r = radeon_ttm_debugfs_init(rdev);
574 if (r) {
575 DRM_ERROR("Failed to init debugfs\n");
576 return r;
577 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200578 return 0;
579}
580
581void radeon_ttm_fini(struct radeon_device *rdev)
582{
Jerome Glisse4c788672009-11-20 14:29:23 +0100583 int r;
584
Jerome Glisse0a0c7592009-12-11 20:36:19 +0100585 if (!rdev->mman.initialized)
586 return;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200587 if (rdev->stollen_vga_memory) {
Jerome Glisse4c788672009-11-20 14:29:23 +0100588 r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
589 if (r == 0) {
590 radeon_bo_unpin(rdev->stollen_vga_memory);
591 radeon_bo_unreserve(rdev->stollen_vga_memory);
592 }
593 radeon_bo_unref(&rdev->stollen_vga_memory);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200594 }
595 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
596 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
597 ttm_bo_device_release(&rdev->mman.bdev);
598 radeon_gart_fini(rdev);
599 radeon_ttm_global_fini(rdev);
Jerome Glisse0a0c7592009-12-11 20:36:19 +0100600 rdev->mman.initialized = false;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200601 DRM_INFO("radeon: ttm finalized\n");
602}
603
604static struct vm_operations_struct radeon_ttm_vm_ops;
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +0400605static const struct vm_operations_struct *ttm_vm_ops = NULL;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200606
607static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
608{
609 struct ttm_buffer_object *bo;
Matthew Garrett5876dd22010-04-26 15:52:20 -0400610 struct radeon_device *rdev;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200611 int r;
612
Matthew Garrett5876dd22010-04-26 15:52:20 -0400613 bo = (struct ttm_buffer_object *)vma->vm_private_data;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200614 if (bo == NULL) {
615 return VM_FAULT_NOPAGE;
616 }
Matthew Garrett5876dd22010-04-26 15:52:20 -0400617 rdev = radeon_get_rdev(bo->bdev);
618 mutex_lock(&rdev->vram_mutex);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200619 r = ttm_vm_ops->fault(vma, vmf);
Matthew Garrett5876dd22010-04-26 15:52:20 -0400620 mutex_unlock(&rdev->vram_mutex);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200621 return r;
622}
623
624int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
625{
626 struct drm_file *file_priv;
627 struct radeon_device *rdev;
628 int r;
629
630 if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
631 return drm_mmap(filp, vma);
632 }
633
634 file_priv = (struct drm_file *)filp->private_data;
635 rdev = file_priv->minor->dev->dev_private;
636 if (rdev == NULL) {
637 return -EINVAL;
638 }
639 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev);
640 if (unlikely(r != 0)) {
641 return r;
642 }
643 if (unlikely(ttm_vm_ops == NULL)) {
644 ttm_vm_ops = vma->vm_ops;
645 radeon_ttm_vm_ops = *ttm_vm_ops;
646 radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
647 }
648 vma->vm_ops = &radeon_ttm_vm_ops;
649 return 0;
650}
651
652
653/*
654 * TTM backend functions.
655 */
656struct radeon_ttm_backend {
657 struct ttm_backend backend;
658 struct radeon_device *rdev;
659 unsigned long num_pages;
660 struct page **pages;
661 struct page *dummy_read_page;
662 bool populated;
663 bool bound;
664 unsigned offset;
665};
666
667static int radeon_ttm_backend_populate(struct ttm_backend *backend,
668 unsigned long num_pages,
669 struct page **pages,
670 struct page *dummy_read_page)
671{
672 struct radeon_ttm_backend *gtt;
673
674 gtt = container_of(backend, struct radeon_ttm_backend, backend);
675 gtt->pages = pages;
676 gtt->num_pages = num_pages;
677 gtt->dummy_read_page = dummy_read_page;
678 gtt->populated = true;
679 return 0;
680}
681
682static void radeon_ttm_backend_clear(struct ttm_backend *backend)
683{
684 struct radeon_ttm_backend *gtt;
685
686 gtt = container_of(backend, struct radeon_ttm_backend, backend);
687 gtt->pages = NULL;
688 gtt->num_pages = 0;
689 gtt->dummy_read_page = NULL;
690 gtt->populated = false;
691 gtt->bound = false;
692}
693
694
695static int radeon_ttm_backend_bind(struct ttm_backend *backend,
696 struct ttm_mem_reg *bo_mem)
697{
698 struct radeon_ttm_backend *gtt;
699 int r;
700
701 gtt = container_of(backend, struct radeon_ttm_backend, backend);
702 gtt->offset = bo_mem->mm_node->start << PAGE_SHIFT;
703 if (!gtt->num_pages) {
704 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", gtt->num_pages, bo_mem, backend);
705 }
706 r = radeon_gart_bind(gtt->rdev, gtt->offset,
707 gtt->num_pages, gtt->pages);
708 if (r) {
709 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
710 gtt->num_pages, gtt->offset);
711 return r;
712 }
713 gtt->bound = true;
714 return 0;
715}
716
717static int radeon_ttm_backend_unbind(struct ttm_backend *backend)
718{
719 struct radeon_ttm_backend *gtt;
720
721 gtt = container_of(backend, struct radeon_ttm_backend, backend);
722 radeon_gart_unbind(gtt->rdev, gtt->offset, gtt->num_pages);
723 gtt->bound = false;
724 return 0;
725}
726
727static void radeon_ttm_backend_destroy(struct ttm_backend *backend)
728{
729 struct radeon_ttm_backend *gtt;
730
731 gtt = container_of(backend, struct radeon_ttm_backend, backend);
732 if (gtt->bound) {
733 radeon_ttm_backend_unbind(backend);
734 }
735 kfree(gtt);
736}
737
738static struct ttm_backend_func radeon_backend_func = {
739 .populate = &radeon_ttm_backend_populate,
740 .clear = &radeon_ttm_backend_clear,
741 .bind = &radeon_ttm_backend_bind,
742 .unbind = &radeon_ttm_backend_unbind,
743 .destroy = &radeon_ttm_backend_destroy,
744};
745
746struct ttm_backend *radeon_ttm_backend_create(struct radeon_device *rdev)
747{
748 struct radeon_ttm_backend *gtt;
749
750 gtt = kzalloc(sizeof(struct radeon_ttm_backend), GFP_KERNEL);
751 if (gtt == NULL) {
752 return NULL;
753 }
754 gtt->backend.bdev = &rdev->mman.bdev;
755 gtt->backend.flags = 0;
756 gtt->backend.func = &radeon_backend_func;
757 gtt->rdev = rdev;
758 gtt->pages = NULL;
759 gtt->num_pages = 0;
760 gtt->dummy_read_page = NULL;
761 gtt->populated = false;
762 gtt->bound = false;
763 return &gtt->backend;
764}
Dave Airliefa8a1232009-08-26 13:13:37 +1000765
766#define RADEON_DEBUGFS_MEM_TYPES 2
767
Dave Airliefa8a1232009-08-26 13:13:37 +1000768#if defined(CONFIG_DEBUG_FS)
769static int radeon_mm_dump_table(struct seq_file *m, void *data)
770{
771 struct drm_info_node *node = (struct drm_info_node *)m->private;
772 struct drm_mm *mm = (struct drm_mm *)node->info_ent->data;
773 struct drm_device *dev = node->minor->dev;
774 struct radeon_device *rdev = dev->dev_private;
775 int ret;
776 struct ttm_bo_global *glob = rdev->mman.bdev.glob;
777
778 spin_lock(&glob->lru_lock);
779 ret = drm_mm_dump_table(m, mm);
780 spin_unlock(&glob->lru_lock);
781 return ret;
782}
783#endif
784
785static int radeon_ttm_debugfs_init(struct radeon_device *rdev)
786{
Mikael Petterssonf4e45d02009-09-28 18:27:23 +0200787#if defined(CONFIG_DEBUG_FS)
Pauli Nieminen8d7cddc2010-04-01 12:44:59 +0000788 static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+1];
789 static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+1][32];
Dave Airliefa8a1232009-08-26 13:13:37 +1000790 unsigned i;
791
Dave Airliefa8a1232009-08-26 13:13:37 +1000792 for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) {
793 if (i == 0)
794 sprintf(radeon_mem_types_names[i], "radeon_vram_mm");
795 else
796 sprintf(radeon_mem_types_names[i], "radeon_gtt_mm");
797 radeon_mem_types_list[i].name = radeon_mem_types_names[i];
798 radeon_mem_types_list[i].show = &radeon_mm_dump_table;
799 radeon_mem_types_list[i].driver_features = 0;
800 if (i == 0)
801 radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_VRAM].manager;
802 else
803 radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager;
804
805 }
Pauli Nieminen8d7cddc2010-04-01 12:44:59 +0000806 /* Add ttm page pool to debugfs */
807 sprintf(radeon_mem_types_names[i], "ttm_page_pool");
808 radeon_mem_types_list[i].name = radeon_mem_types_names[i];
809 radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs;
810 radeon_mem_types_list[i].driver_features = 0;
811 radeon_mem_types_list[i].data = NULL;
812 return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES+1);
Dave Airliefa8a1232009-08-26 13:13:37 +1000813
814#endif
815 return 0;
816}