Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2007 Dave Airlied |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 6 | * copy of this software and associated documentation files (the "Software"), |
| 7 | * to deal in the Software without restriction, including without limitation |
| 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 9 | * and/or sell copies of the Software, and to permit persons to whom the |
| 10 | * Software is furnished to do so, subject to the following conditions: |
| 11 | * |
| 12 | * The above copyright notice and this permission notice (including the next |
| 13 | * paragraph) shall be included in all copies or substantial portions of the |
| 14 | * Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 22 | * OTHER DEALINGS IN THE SOFTWARE. |
| 23 | */ |
| 24 | /* |
| 25 | * Authors: Dave Airlied <airlied@linux.ie> |
| 26 | * Ben Skeggs <darktama@iinet.net.au> |
| 27 | * Jeremy Kolb <jkolb@brandeis.edu> |
| 28 | */ |
| 29 | |
| 30 | #include "drmP.h" |
| 31 | |
| 32 | #include "nouveau_drm.h" |
| 33 | #include "nouveau_drv.h" |
| 34 | #include "nouveau_dma.h" |
| 35 | |
| 36 | static void |
| 37 | nouveau_bo_del_ttm(struct ttm_buffer_object *bo) |
| 38 | { |
| 39 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
| 40 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
| 41 | |
| 42 | ttm_bo_kunmap(&nvbo->kmap); |
| 43 | |
| 44 | if (unlikely(nvbo->gem)) |
| 45 | DRM_ERROR("bo %p still attached to GEM object\n", bo); |
| 46 | |
| 47 | spin_lock(&dev_priv->ttm.bo_list_lock); |
| 48 | list_del(&nvbo->head); |
| 49 | spin_unlock(&dev_priv->ttm.bo_list_lock); |
| 50 | kfree(nvbo); |
| 51 | } |
| 52 | |
| 53 | int |
| 54 | nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, |
| 55 | int size, int align, uint32_t flags, uint32_t tile_mode, |
| 56 | uint32_t tile_flags, bool no_vm, bool mappable, |
| 57 | struct nouveau_bo **pnvbo) |
| 58 | { |
| 59 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
| 60 | struct nouveau_bo *nvbo; |
| 61 | int ret, n = 0; |
| 62 | |
| 63 | nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); |
| 64 | if (!nvbo) |
| 65 | return -ENOMEM; |
| 66 | INIT_LIST_HEAD(&nvbo->head); |
| 67 | INIT_LIST_HEAD(&nvbo->entry); |
| 68 | nvbo->mappable = mappable; |
| 69 | nvbo->no_vm = no_vm; |
| 70 | nvbo->tile_mode = tile_mode; |
| 71 | nvbo->tile_flags = tile_flags; |
| 72 | |
| 73 | /* |
| 74 | * Some of the tile_flags have a periodic structure of N*4096 bytes, |
| 75 | * align to to that as well as the page size. Overallocate memory to |
| 76 | * avoid corruption of other buffer objects. |
| 77 | */ |
| 78 | switch (tile_flags) { |
| 79 | case 0x1800: |
| 80 | case 0x2800: |
| 81 | case 0x4800: |
| 82 | case 0x7a00: |
| 83 | if (dev_priv->chipset >= 0xA0) { |
| 84 | /* This is based on high end cards with 448 bits |
| 85 | * memory bus, could be different elsewhere.*/ |
| 86 | size += 6 * 28672; |
| 87 | /* 8 * 28672 is the actual alignment requirement, |
| 88 | * but we must also align to page size. */ |
| 89 | align = 2 * 8 * 28672; |
| 90 | } else if (dev_priv->chipset >= 0x90) { |
| 91 | size += 3 * 16384; |
| 92 | align = 12 * 16834; |
| 93 | } else { |
| 94 | size += 3 * 8192; |
| 95 | /* 12 * 8192 is the actual alignment requirement, |
| 96 | * but we must also align to page size. */ |
| 97 | align = 2 * 12 * 8192; |
| 98 | } |
| 99 | break; |
| 100 | default: |
| 101 | break; |
| 102 | } |
| 103 | |
| 104 | align >>= PAGE_SHIFT; |
| 105 | |
| 106 | size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); |
| 107 | if (dev_priv->card_type == NV_50) { |
| 108 | size = (size + 65535) & ~65535; |
| 109 | if (align < (65536 / PAGE_SIZE)) |
| 110 | align = (65536 / PAGE_SIZE); |
| 111 | } |
| 112 | |
| 113 | if (flags & TTM_PL_FLAG_VRAM) |
| 114 | nvbo->placements[n++] = TTM_PL_FLAG_VRAM | TTM_PL_MASK_CACHING; |
| 115 | if (flags & TTM_PL_FLAG_TT) |
| 116 | nvbo->placements[n++] = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; |
| 117 | nvbo->placement.fpfn = 0; |
| 118 | nvbo->placement.lpfn = mappable ? dev_priv->fb_mappable_pages : 0; |
| 119 | nvbo->placement.placement = nvbo->placements; |
| 120 | nvbo->placement.busy_placement = nvbo->placements; |
| 121 | nvbo->placement.num_placement = n; |
| 122 | nvbo->placement.num_busy_placement = n; |
| 123 | |
| 124 | nvbo->channel = chan; |
| 125 | nouveau_bo_placement_set(nvbo, flags); |
| 126 | ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, |
| 127 | ttm_bo_type_device, &nvbo->placement, align, 0, |
| 128 | false, NULL, size, nouveau_bo_del_ttm); |
| 129 | nvbo->channel = NULL; |
| 130 | if (ret) { |
| 131 | /* ttm will call nouveau_bo_del_ttm if it fails.. */ |
| 132 | return ret; |
| 133 | } |
| 134 | |
| 135 | spin_lock(&dev_priv->ttm.bo_list_lock); |
| 136 | list_add_tail(&nvbo->head, &dev_priv->ttm.bo_list); |
| 137 | spin_unlock(&dev_priv->ttm.bo_list_lock); |
| 138 | *pnvbo = nvbo; |
| 139 | return 0; |
| 140 | } |
| 141 | |
| 142 | void |
| 143 | nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t memtype) |
| 144 | { |
| 145 | int n = 0; |
| 146 | |
| 147 | if (memtype & TTM_PL_FLAG_VRAM) |
| 148 | nvbo->placements[n++] = TTM_PL_FLAG_VRAM | TTM_PL_MASK_CACHING; |
| 149 | if (memtype & TTM_PL_FLAG_TT) |
| 150 | nvbo->placements[n++] = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; |
| 151 | if (memtype & TTM_PL_FLAG_SYSTEM) |
| 152 | nvbo->placements[n++] = TTM_PL_FLAG_SYSTEM | TTM_PL_MASK_CACHING; |
| 153 | nvbo->placement.placement = nvbo->placements; |
| 154 | nvbo->placement.busy_placement = nvbo->placements; |
| 155 | nvbo->placement.num_placement = n; |
| 156 | nvbo->placement.num_busy_placement = n; |
| 157 | } |
| 158 | |
| 159 | int |
| 160 | nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) |
| 161 | { |
| 162 | struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); |
| 163 | struct ttm_buffer_object *bo = &nvbo->bo; |
| 164 | int ret, i; |
| 165 | |
| 166 | if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { |
| 167 | NV_ERROR(nouveau_bdev(bo->bdev)->dev, |
| 168 | "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, |
| 169 | 1 << bo->mem.mem_type, memtype); |
| 170 | return -EINVAL; |
| 171 | } |
| 172 | |
| 173 | if (nvbo->pin_refcnt++) |
| 174 | return 0; |
| 175 | |
| 176 | ret = ttm_bo_reserve(bo, false, false, false, 0); |
| 177 | if (ret) |
| 178 | goto out; |
| 179 | |
| 180 | nouveau_bo_placement_set(nvbo, memtype); |
| 181 | for (i = 0; i < nvbo->placement.num_placement; i++) |
| 182 | nvbo->placements[i] |= TTM_PL_FLAG_NO_EVICT; |
| 183 | |
| 184 | ret = ttm_bo_validate(bo, &nvbo->placement, false, false); |
| 185 | if (ret == 0) { |
| 186 | switch (bo->mem.mem_type) { |
| 187 | case TTM_PL_VRAM: |
| 188 | dev_priv->fb_aper_free -= bo->mem.size; |
| 189 | break; |
| 190 | case TTM_PL_TT: |
| 191 | dev_priv->gart_info.aper_free -= bo->mem.size; |
| 192 | break; |
| 193 | default: |
| 194 | break; |
| 195 | } |
| 196 | } |
| 197 | ttm_bo_unreserve(bo); |
| 198 | out: |
| 199 | if (unlikely(ret)) |
| 200 | nvbo->pin_refcnt--; |
| 201 | return ret; |
| 202 | } |
| 203 | |
| 204 | int |
| 205 | nouveau_bo_unpin(struct nouveau_bo *nvbo) |
| 206 | { |
| 207 | struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); |
| 208 | struct ttm_buffer_object *bo = &nvbo->bo; |
| 209 | int ret, i; |
| 210 | |
| 211 | if (--nvbo->pin_refcnt) |
| 212 | return 0; |
| 213 | |
| 214 | ret = ttm_bo_reserve(bo, false, false, false, 0); |
| 215 | if (ret) |
| 216 | return ret; |
| 217 | |
| 218 | for (i = 0; i < nvbo->placement.num_placement; i++) |
| 219 | nvbo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT; |
| 220 | |
| 221 | ret = ttm_bo_validate(bo, &nvbo->placement, false, false); |
| 222 | if (ret == 0) { |
| 223 | switch (bo->mem.mem_type) { |
| 224 | case TTM_PL_VRAM: |
| 225 | dev_priv->fb_aper_free += bo->mem.size; |
| 226 | break; |
| 227 | case TTM_PL_TT: |
| 228 | dev_priv->gart_info.aper_free += bo->mem.size; |
| 229 | break; |
| 230 | default: |
| 231 | break; |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | ttm_bo_unreserve(bo); |
| 236 | return ret; |
| 237 | } |
| 238 | |
| 239 | int |
| 240 | nouveau_bo_map(struct nouveau_bo *nvbo) |
| 241 | { |
| 242 | int ret; |
| 243 | |
| 244 | ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); |
| 245 | if (ret) |
| 246 | return ret; |
| 247 | |
| 248 | ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); |
| 249 | ttm_bo_unreserve(&nvbo->bo); |
| 250 | return ret; |
| 251 | } |
| 252 | |
| 253 | void |
| 254 | nouveau_bo_unmap(struct nouveau_bo *nvbo) |
| 255 | { |
| 256 | ttm_bo_kunmap(&nvbo->kmap); |
| 257 | } |
| 258 | |
| 259 | u16 |
| 260 | nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) |
| 261 | { |
| 262 | bool is_iomem; |
| 263 | u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); |
| 264 | mem = &mem[index]; |
| 265 | if (is_iomem) |
| 266 | return ioread16_native((void __force __iomem *)mem); |
| 267 | else |
| 268 | return *mem; |
| 269 | } |
| 270 | |
| 271 | void |
| 272 | nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) |
| 273 | { |
| 274 | bool is_iomem; |
| 275 | u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); |
| 276 | mem = &mem[index]; |
| 277 | if (is_iomem) |
| 278 | iowrite16_native(val, (void __force __iomem *)mem); |
| 279 | else |
| 280 | *mem = val; |
| 281 | } |
| 282 | |
| 283 | u32 |
| 284 | nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) |
| 285 | { |
| 286 | bool is_iomem; |
| 287 | u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); |
| 288 | mem = &mem[index]; |
| 289 | if (is_iomem) |
| 290 | return ioread32_native((void __force __iomem *)mem); |
| 291 | else |
| 292 | return *mem; |
| 293 | } |
| 294 | |
| 295 | void |
| 296 | nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) |
| 297 | { |
| 298 | bool is_iomem; |
| 299 | u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); |
| 300 | mem = &mem[index]; |
| 301 | if (is_iomem) |
| 302 | iowrite32_native(val, (void __force __iomem *)mem); |
| 303 | else |
| 304 | *mem = val; |
| 305 | } |
| 306 | |
| 307 | static struct ttm_backend * |
| 308 | nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev) |
| 309 | { |
| 310 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); |
| 311 | struct drm_device *dev = dev_priv->dev; |
| 312 | |
| 313 | switch (dev_priv->gart_info.type) { |
Ben Skeggs | b694dfb | 2009-12-15 10:38:32 +1000 | [diff] [blame] | 314 | #if __OS_HAS_AGP |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 315 | case NOUVEAU_GART_AGP: |
| 316 | return ttm_agp_backend_init(bdev, dev->agp->bridge); |
Ben Skeggs | b694dfb | 2009-12-15 10:38:32 +1000 | [diff] [blame] | 317 | #endif |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 318 | case NOUVEAU_GART_SGDMA: |
| 319 | return nouveau_sgdma_init_ttm(dev); |
| 320 | default: |
| 321 | NV_ERROR(dev, "Unknown GART type %d\n", |
| 322 | dev_priv->gart_info.type); |
| 323 | break; |
| 324 | } |
| 325 | |
| 326 | return NULL; |
| 327 | } |
| 328 | |
| 329 | static int |
| 330 | nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) |
| 331 | { |
| 332 | /* We'll do this from user space. */ |
| 333 | return 0; |
| 334 | } |
| 335 | |
| 336 | static int |
| 337 | nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, |
| 338 | struct ttm_mem_type_manager *man) |
| 339 | { |
| 340 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); |
| 341 | struct drm_device *dev = dev_priv->dev; |
| 342 | |
| 343 | switch (type) { |
| 344 | case TTM_PL_SYSTEM: |
| 345 | man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; |
| 346 | man->available_caching = TTM_PL_MASK_CACHING; |
| 347 | man->default_caching = TTM_PL_FLAG_CACHED; |
| 348 | break; |
| 349 | case TTM_PL_VRAM: |
| 350 | man->flags = TTM_MEMTYPE_FLAG_FIXED | |
| 351 | TTM_MEMTYPE_FLAG_MAPPABLE | |
| 352 | TTM_MEMTYPE_FLAG_NEEDS_IOREMAP; |
| 353 | man->available_caching = TTM_PL_FLAG_UNCACHED | |
| 354 | TTM_PL_FLAG_WC; |
| 355 | man->default_caching = TTM_PL_FLAG_WC; |
| 356 | |
| 357 | man->io_addr = NULL; |
| 358 | man->io_offset = drm_get_resource_start(dev, 1); |
| 359 | man->io_size = drm_get_resource_len(dev, 1); |
| 360 | if (man->io_size > nouveau_mem_fb_amount(dev)) |
| 361 | man->io_size = nouveau_mem_fb_amount(dev); |
| 362 | |
| 363 | man->gpu_offset = dev_priv->vm_vram_base; |
| 364 | break; |
| 365 | case TTM_PL_TT: |
| 366 | switch (dev_priv->gart_info.type) { |
| 367 | case NOUVEAU_GART_AGP: |
| 368 | man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | |
| 369 | TTM_MEMTYPE_FLAG_NEEDS_IOREMAP; |
| 370 | man->available_caching = TTM_PL_FLAG_UNCACHED; |
| 371 | man->default_caching = TTM_PL_FLAG_UNCACHED; |
| 372 | break; |
| 373 | case NOUVEAU_GART_SGDMA: |
| 374 | man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | |
| 375 | TTM_MEMTYPE_FLAG_CMA; |
| 376 | man->available_caching = TTM_PL_MASK_CACHING; |
| 377 | man->default_caching = TTM_PL_FLAG_CACHED; |
| 378 | break; |
| 379 | default: |
| 380 | NV_ERROR(dev, "Unknown GART type: %d\n", |
| 381 | dev_priv->gart_info.type); |
| 382 | return -EINVAL; |
| 383 | } |
| 384 | |
| 385 | man->io_offset = dev_priv->gart_info.aper_base; |
| 386 | man->io_size = dev_priv->gart_info.aper_size; |
| 387 | man->io_addr = NULL; |
| 388 | man->gpu_offset = dev_priv->vm_gart_base; |
| 389 | break; |
| 390 | default: |
| 391 | NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); |
| 392 | return -EINVAL; |
| 393 | } |
| 394 | return 0; |
| 395 | } |
| 396 | |
| 397 | static void |
| 398 | nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) |
| 399 | { |
| 400 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
| 401 | |
| 402 | switch (bo->mem.mem_type) { |
| 403 | default: |
| 404 | nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM); |
| 405 | break; |
| 406 | } |
| 407 | } |
| 408 | |
| 409 | |
| 410 | /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access |
| 411 | * TTM_PL_{VRAM,TT} directly. |
| 412 | */ |
| 413 | static int |
| 414 | nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, |
| 415 | struct nouveau_bo *nvbo, bool evict, bool no_wait, |
| 416 | struct ttm_mem_reg *new_mem) |
| 417 | { |
| 418 | struct nouveau_fence *fence = NULL; |
| 419 | int ret; |
| 420 | |
| 421 | ret = nouveau_fence_new(chan, &fence, true); |
| 422 | if (ret) |
| 423 | return ret; |
| 424 | |
| 425 | ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, |
| 426 | evict, no_wait, new_mem); |
| 427 | nouveau_fence_unref((void *)&fence); |
| 428 | return ret; |
| 429 | } |
| 430 | |
| 431 | static inline uint32_t |
| 432 | nouveau_bo_mem_ctxdma(struct nouveau_bo *nvbo, struct nouveau_channel *chan, |
| 433 | struct ttm_mem_reg *mem) |
| 434 | { |
| 435 | if (chan == nouveau_bdev(nvbo->bo.bdev)->channel) { |
| 436 | if (mem->mem_type == TTM_PL_TT) |
| 437 | return NvDmaGART; |
| 438 | return NvDmaVRAM; |
| 439 | } |
| 440 | |
| 441 | if (mem->mem_type == TTM_PL_TT) |
| 442 | return chan->gart_handle; |
| 443 | return chan->vram_handle; |
| 444 | } |
| 445 | |
| 446 | static int |
| 447 | nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, int no_wait, |
| 448 | struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) |
| 449 | { |
| 450 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
| 451 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
| 452 | struct nouveau_channel *chan; |
| 453 | uint64_t src_offset, dst_offset; |
| 454 | uint32_t page_count; |
| 455 | int ret; |
| 456 | |
| 457 | chan = nvbo->channel; |
| 458 | if (!chan || nvbo->tile_flags || nvbo->no_vm) { |
| 459 | chan = dev_priv->channel; |
| 460 | if (!chan) |
| 461 | return -EINVAL; |
| 462 | } |
| 463 | |
| 464 | src_offset = old_mem->mm_node->start << PAGE_SHIFT; |
| 465 | dst_offset = new_mem->mm_node->start << PAGE_SHIFT; |
| 466 | if (chan != dev_priv->channel) { |
| 467 | if (old_mem->mem_type == TTM_PL_TT) |
| 468 | src_offset += dev_priv->vm_gart_base; |
| 469 | else |
| 470 | src_offset += dev_priv->vm_vram_base; |
| 471 | |
| 472 | if (new_mem->mem_type == TTM_PL_TT) |
| 473 | dst_offset += dev_priv->vm_gart_base; |
| 474 | else |
| 475 | dst_offset += dev_priv->vm_vram_base; |
| 476 | } |
| 477 | |
| 478 | ret = RING_SPACE(chan, 3); |
| 479 | if (ret) |
| 480 | return ret; |
| 481 | BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2); |
| 482 | OUT_RING(chan, nouveau_bo_mem_ctxdma(nvbo, chan, old_mem)); |
| 483 | OUT_RING(chan, nouveau_bo_mem_ctxdma(nvbo, chan, new_mem)); |
| 484 | |
| 485 | if (dev_priv->card_type >= NV_50) { |
| 486 | ret = RING_SPACE(chan, 4); |
| 487 | if (ret) |
| 488 | return ret; |
| 489 | BEGIN_RING(chan, NvSubM2MF, 0x0200, 1); |
| 490 | OUT_RING(chan, 1); |
| 491 | BEGIN_RING(chan, NvSubM2MF, 0x021c, 1); |
| 492 | OUT_RING(chan, 1); |
| 493 | } |
| 494 | |
| 495 | page_count = new_mem->num_pages; |
| 496 | while (page_count) { |
| 497 | int line_count = (page_count > 2047) ? 2047 : page_count; |
| 498 | |
| 499 | if (dev_priv->card_type >= NV_50) { |
| 500 | ret = RING_SPACE(chan, 3); |
| 501 | if (ret) |
| 502 | return ret; |
| 503 | BEGIN_RING(chan, NvSubM2MF, 0x0238, 2); |
| 504 | OUT_RING(chan, upper_32_bits(src_offset)); |
| 505 | OUT_RING(chan, upper_32_bits(dst_offset)); |
| 506 | } |
| 507 | ret = RING_SPACE(chan, 11); |
| 508 | if (ret) |
| 509 | return ret; |
| 510 | BEGIN_RING(chan, NvSubM2MF, |
| 511 | NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); |
| 512 | OUT_RING(chan, lower_32_bits(src_offset)); |
| 513 | OUT_RING(chan, lower_32_bits(dst_offset)); |
| 514 | OUT_RING(chan, PAGE_SIZE); /* src_pitch */ |
| 515 | OUT_RING(chan, PAGE_SIZE); /* dst_pitch */ |
| 516 | OUT_RING(chan, PAGE_SIZE); /* line_length */ |
| 517 | OUT_RING(chan, line_count); |
| 518 | OUT_RING(chan, (1<<8)|(1<<0)); |
| 519 | OUT_RING(chan, 0); |
| 520 | BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); |
| 521 | OUT_RING(chan, 0); |
| 522 | |
| 523 | page_count -= line_count; |
| 524 | src_offset += (PAGE_SIZE * line_count); |
| 525 | dst_offset += (PAGE_SIZE * line_count); |
| 526 | } |
| 527 | |
| 528 | return nouveau_bo_move_accel_cleanup(chan, nvbo, evict, no_wait, new_mem); |
| 529 | } |
| 530 | |
| 531 | static int |
| 532 | nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, |
| 533 | bool no_wait, struct ttm_mem_reg *new_mem) |
| 534 | { |
| 535 | u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; |
| 536 | struct ttm_placement placement; |
| 537 | struct ttm_mem_reg tmp_mem; |
| 538 | int ret; |
| 539 | |
| 540 | placement.fpfn = placement.lpfn = 0; |
| 541 | placement.num_placement = placement.num_busy_placement = 1; |
| 542 | placement.placement = &placement_memtype; |
| 543 | |
| 544 | tmp_mem = *new_mem; |
| 545 | tmp_mem.mm_node = NULL; |
| 546 | ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait); |
| 547 | if (ret) |
| 548 | return ret; |
| 549 | |
| 550 | ret = ttm_tt_bind(bo->ttm, &tmp_mem); |
| 551 | if (ret) |
| 552 | goto out; |
| 553 | |
| 554 | ret = nouveau_bo_move_m2mf(bo, true, no_wait, &bo->mem, &tmp_mem); |
| 555 | if (ret) |
| 556 | goto out; |
| 557 | |
| 558 | ret = ttm_bo_move_ttm(bo, evict, no_wait, new_mem); |
| 559 | out: |
| 560 | if (tmp_mem.mm_node) { |
| 561 | spin_lock(&bo->bdev->glob->lru_lock); |
| 562 | drm_mm_put_block(tmp_mem.mm_node); |
| 563 | spin_unlock(&bo->bdev->glob->lru_lock); |
| 564 | } |
| 565 | |
| 566 | return ret; |
| 567 | } |
| 568 | |
| 569 | static int |
| 570 | nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, |
| 571 | bool no_wait, struct ttm_mem_reg *new_mem) |
| 572 | { |
| 573 | u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; |
| 574 | struct ttm_placement placement; |
| 575 | struct ttm_mem_reg tmp_mem; |
| 576 | int ret; |
| 577 | |
| 578 | placement.fpfn = placement.lpfn = 0; |
| 579 | placement.num_placement = placement.num_busy_placement = 1; |
| 580 | placement.placement = &placement_memtype; |
| 581 | |
| 582 | tmp_mem = *new_mem; |
| 583 | tmp_mem.mm_node = NULL; |
| 584 | ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait); |
| 585 | if (ret) |
| 586 | return ret; |
| 587 | |
| 588 | ret = ttm_bo_move_ttm(bo, evict, no_wait, &tmp_mem); |
| 589 | if (ret) |
| 590 | goto out; |
| 591 | |
| 592 | ret = nouveau_bo_move_m2mf(bo, true, no_wait, &bo->mem, new_mem); |
| 593 | if (ret) |
| 594 | goto out; |
| 595 | |
| 596 | out: |
| 597 | if (tmp_mem.mm_node) { |
| 598 | spin_lock(&bo->bdev->glob->lru_lock); |
| 599 | drm_mm_put_block(tmp_mem.mm_node); |
| 600 | spin_unlock(&bo->bdev->glob->lru_lock); |
| 601 | } |
| 602 | |
| 603 | return ret; |
| 604 | } |
| 605 | |
| 606 | static int |
| 607 | nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, |
| 608 | bool no_wait, struct ttm_mem_reg *new_mem) |
| 609 | { |
| 610 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
| 611 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
| 612 | struct drm_device *dev = dev_priv->dev; |
| 613 | struct ttm_mem_reg *old_mem = &bo->mem; |
| 614 | int ret; |
| 615 | |
| 616 | if (dev_priv->card_type == NV_50 && new_mem->mem_type == TTM_PL_VRAM && |
| 617 | !nvbo->no_vm) { |
| 618 | uint64_t offset = new_mem->mm_node->start << PAGE_SHIFT; |
| 619 | |
| 620 | ret = nv50_mem_vm_bind_linear(dev, |
| 621 | offset + dev_priv->vm_vram_base, |
| 622 | new_mem->size, nvbo->tile_flags, |
| 623 | offset); |
| 624 | if (ret) |
| 625 | return ret; |
| 626 | } |
| 627 | |
| 628 | if (dev_priv->init_state != NOUVEAU_CARD_INIT_DONE) |
| 629 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
| 630 | |
| 631 | if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { |
| 632 | BUG_ON(bo->mem.mm_node != NULL); |
| 633 | bo->mem = *new_mem; |
| 634 | new_mem->mm_node = NULL; |
| 635 | return 0; |
| 636 | } |
| 637 | |
| 638 | if (new_mem->mem_type == TTM_PL_SYSTEM) { |
| 639 | if (old_mem->mem_type == TTM_PL_SYSTEM) |
| 640 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
| 641 | if (nouveau_bo_move_flipd(bo, evict, intr, no_wait, new_mem)) |
| 642 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
| 643 | } else if (old_mem->mem_type == TTM_PL_SYSTEM) { |
| 644 | if (nouveau_bo_move_flips(bo, evict, intr, no_wait, new_mem)) |
| 645 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
| 646 | } else { |
| 647 | if (nouveau_bo_move_m2mf(bo, evict, no_wait, old_mem, new_mem)) |
| 648 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
| 649 | } |
| 650 | |
| 651 | return 0; |
| 652 | } |
| 653 | |
| 654 | static int |
| 655 | nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) |
| 656 | { |
| 657 | return 0; |
| 658 | } |
| 659 | |
| 660 | struct ttm_bo_driver nouveau_bo_driver = { |
| 661 | .create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry, |
| 662 | .invalidate_caches = nouveau_bo_invalidate_caches, |
| 663 | .init_mem_type = nouveau_bo_init_mem_type, |
| 664 | .evict_flags = nouveau_bo_evict_flags, |
| 665 | .move = nouveau_bo_move, |
| 666 | .verify_access = nouveau_bo_verify_access, |
| 667 | .sync_obj_signaled = nouveau_fence_signalled, |
| 668 | .sync_obj_wait = nouveau_fence_wait, |
| 669 | .sync_obj_flush = nouveau_fence_flush, |
| 670 | .sync_obj_unref = nouveau_fence_unref, |
| 671 | .sync_obj_ref = nouveau_fence_ref, |
| 672 | }; |
| 673 | |