Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2008 Jerome Glisse. |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 6 | * copy of this software and associated documentation files (the "Software"), |
| 7 | * to deal in the Software without restriction, including without limitation |
| 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 9 | * and/or sell copies of the Software, and to permit persons to whom the |
| 10 | * Software is furnished to do so, subject to the following conditions: |
| 11 | * |
| 12 | * The above copyright notice and this permission notice (including the next |
| 13 | * paragraph) shall be included in all copies or substantial portions of the |
| 14 | * Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| 22 | * DEALINGS IN THE SOFTWARE. |
| 23 | * |
| 24 | * Authors: |
| 25 | * Jerome Glisse <glisse@freedesktop.org> |
| 26 | */ |
| 27 | #include <linux/list_sort.h> |
| 28 | #include <drm/drmP.h> |
| 29 | #include <drm/amdgpu_drm.h> |
| 30 | #include "amdgpu.h" |
| 31 | #include "amdgpu_trace.h" |
| 32 | |
| 33 | #define AMDGPU_CS_MAX_PRIORITY 32u |
| 34 | #define AMDGPU_CS_NUM_BUCKETS (AMDGPU_CS_MAX_PRIORITY + 1) |
| 35 | |
| 36 | /* This is based on the bucket sort with O(n) time complexity. |
| 37 | * An item with priority "i" is added to bucket[i]. The lists are then |
| 38 | * concatenated in descending order. |
| 39 | */ |
| 40 | struct amdgpu_cs_buckets { |
| 41 | struct list_head bucket[AMDGPU_CS_NUM_BUCKETS]; |
| 42 | }; |
| 43 | |
| 44 | static void amdgpu_cs_buckets_init(struct amdgpu_cs_buckets *b) |
| 45 | { |
| 46 | unsigned i; |
| 47 | |
| 48 | for (i = 0; i < AMDGPU_CS_NUM_BUCKETS; i++) |
| 49 | INIT_LIST_HEAD(&b->bucket[i]); |
| 50 | } |
| 51 | |
| 52 | static void amdgpu_cs_buckets_add(struct amdgpu_cs_buckets *b, |
| 53 | struct list_head *item, unsigned priority) |
| 54 | { |
| 55 | /* Since buffers which appear sooner in the relocation list are |
| 56 | * likely to be used more often than buffers which appear later |
| 57 | * in the list, the sort mustn't change the ordering of buffers |
| 58 | * with the same priority, i.e. it must be stable. |
| 59 | */ |
| 60 | list_add_tail(item, &b->bucket[min(priority, AMDGPU_CS_MAX_PRIORITY)]); |
| 61 | } |
| 62 | |
| 63 | static void amdgpu_cs_buckets_get_list(struct amdgpu_cs_buckets *b, |
| 64 | struct list_head *out_list) |
| 65 | { |
| 66 | unsigned i; |
| 67 | |
| 68 | /* Connect the sorted buckets in the output list. */ |
| 69 | for (i = 0; i < AMDGPU_CS_NUM_BUCKETS; i++) { |
| 70 | list_splice(&b->bucket[i], out_list); |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, |
| 75 | u32 ip_instance, u32 ring, |
| 76 | struct amdgpu_ring **out_ring) |
| 77 | { |
| 78 | /* Right now all IPs have only one instance - multiple rings. */ |
| 79 | if (ip_instance != 0) { |
| 80 | DRM_ERROR("invalid ip instance: %d\n", ip_instance); |
| 81 | return -EINVAL; |
| 82 | } |
| 83 | |
| 84 | switch (ip_type) { |
| 85 | default: |
| 86 | DRM_ERROR("unknown ip type: %d\n", ip_type); |
| 87 | return -EINVAL; |
| 88 | case AMDGPU_HW_IP_GFX: |
| 89 | if (ring < adev->gfx.num_gfx_rings) { |
| 90 | *out_ring = &adev->gfx.gfx_ring[ring]; |
| 91 | } else { |
| 92 | DRM_ERROR("only %d gfx rings are supported now\n", |
| 93 | adev->gfx.num_gfx_rings); |
| 94 | return -EINVAL; |
| 95 | } |
| 96 | break; |
| 97 | case AMDGPU_HW_IP_COMPUTE: |
| 98 | if (ring < adev->gfx.num_compute_rings) { |
| 99 | *out_ring = &adev->gfx.compute_ring[ring]; |
| 100 | } else { |
| 101 | DRM_ERROR("only %d compute rings are supported now\n", |
| 102 | adev->gfx.num_compute_rings); |
| 103 | return -EINVAL; |
| 104 | } |
| 105 | break; |
| 106 | case AMDGPU_HW_IP_DMA: |
Alex Deucher | c113ea1 | 2015-10-08 16:30:37 -0400 | [diff] [blame] | 107 | if (ring < adev->sdma.num_instances) { |
| 108 | *out_ring = &adev->sdma.instance[ring].ring; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 109 | } else { |
Alex Deucher | c113ea1 | 2015-10-08 16:30:37 -0400 | [diff] [blame] | 110 | DRM_ERROR("only %d SDMA rings are supported\n", |
| 111 | adev->sdma.num_instances); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 112 | return -EINVAL; |
| 113 | } |
| 114 | break; |
| 115 | case AMDGPU_HW_IP_UVD: |
| 116 | *out_ring = &adev->uvd.ring; |
| 117 | break; |
| 118 | case AMDGPU_HW_IP_VCE: |
| 119 | if (ring < 2){ |
| 120 | *out_ring = &adev->vce.ring[ring]; |
| 121 | } else { |
| 122 | DRM_ERROR("only two VCE rings are supported\n"); |
| 123 | return -EINVAL; |
| 124 | } |
| 125 | break; |
| 126 | } |
| 127 | return 0; |
| 128 | } |
| 129 | |
| 130 | int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) |
| 131 | { |
| 132 | union drm_amdgpu_cs *cs = data; |
| 133 | uint64_t *chunk_array_user; |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 134 | uint64_t *chunk_array; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 135 | struct amdgpu_fpriv *fpriv = p->filp->driver_priv; |
Dan Carpenter | 5431350 | 2015-09-25 14:36:55 +0300 | [diff] [blame] | 136 | unsigned size; |
| 137 | int i; |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 138 | int ret; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 139 | |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 140 | if (cs->in.num_chunks == 0) |
| 141 | return 0; |
| 142 | |
| 143 | chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); |
| 144 | if (!chunk_array) |
| 145 | return -ENOMEM; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 146 | |
Christian König | 3cb485f | 2015-05-11 15:34:59 +0200 | [diff] [blame] | 147 | p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); |
| 148 | if (!p->ctx) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 149 | ret = -EINVAL; |
| 150 | goto free_chunk; |
Christian König | 3cb485f | 2015-05-11 15:34:59 +0200 | [diff] [blame] | 151 | } |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 152 | |
Chunming Zhou | a3348bb | 2015-08-18 16:25:46 +0800 | [diff] [blame] | 153 | p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 154 | |
| 155 | /* get chunks */ |
| 156 | INIT_LIST_HEAD(&p->validated); |
Arnd Bergmann | 028423b | 2015-10-07 09:41:27 +0200 | [diff] [blame] | 157 | chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 158 | if (copy_from_user(chunk_array, chunk_array_user, |
| 159 | sizeof(uint64_t)*cs->in.num_chunks)) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 160 | ret = -EFAULT; |
| 161 | goto put_bo_list; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 162 | } |
| 163 | |
| 164 | p->nchunks = cs->in.num_chunks; |
monk.liu | e60b344 | 2015-07-17 18:39:25 +0800 | [diff] [blame] | 165 | p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 166 | GFP_KERNEL); |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 167 | if (!p->chunks) { |
| 168 | ret = -ENOMEM; |
| 169 | goto put_bo_list; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | for (i = 0; i < p->nchunks; i++) { |
| 173 | struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; |
| 174 | struct drm_amdgpu_cs_chunk user_chunk; |
| 175 | uint32_t __user *cdata; |
| 176 | |
Arnd Bergmann | 028423b | 2015-10-07 09:41:27 +0200 | [diff] [blame] | 177 | chunk_ptr = (void __user *)(unsigned long)chunk_array[i]; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 178 | if (copy_from_user(&user_chunk, chunk_ptr, |
| 179 | sizeof(struct drm_amdgpu_cs_chunk))) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 180 | ret = -EFAULT; |
| 181 | i--; |
| 182 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 183 | } |
| 184 | p->chunks[i].chunk_id = user_chunk.chunk_id; |
| 185 | p->chunks[i].length_dw = user_chunk.length_dw; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 186 | |
| 187 | size = p->chunks[i].length_dw; |
Arnd Bergmann | 028423b | 2015-10-07 09:41:27 +0200 | [diff] [blame] | 188 | cdata = (void __user *)(unsigned long)user_chunk.chunk_data; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 189 | p->chunks[i].user_ptr = cdata; |
| 190 | |
| 191 | p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); |
| 192 | if (p->chunks[i].kdata == NULL) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 193 | ret = -ENOMEM; |
| 194 | i--; |
| 195 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 196 | } |
| 197 | size *= sizeof(uint32_t); |
| 198 | if (copy_from_user(p->chunks[i].kdata, cdata, size)) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 199 | ret = -EFAULT; |
| 200 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 201 | } |
| 202 | |
Christian König | 9a5e8fb | 2015-06-23 17:07:03 +0200 | [diff] [blame] | 203 | switch (p->chunks[i].chunk_id) { |
| 204 | case AMDGPU_CHUNK_ID_IB: |
| 205 | p->num_ibs++; |
| 206 | break; |
| 207 | |
| 208 | case AMDGPU_CHUNK_ID_FENCE: |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 209 | size = sizeof(struct drm_amdgpu_cs_chunk_fence); |
| 210 | if (p->chunks[i].length_dw * sizeof(uint32_t) >= size) { |
| 211 | uint32_t handle; |
| 212 | struct drm_gem_object *gobj; |
| 213 | struct drm_amdgpu_cs_chunk_fence *fence_data; |
| 214 | |
| 215 | fence_data = (void *)p->chunks[i].kdata; |
| 216 | handle = fence_data->handle; |
| 217 | gobj = drm_gem_object_lookup(p->adev->ddev, |
| 218 | p->filp, handle); |
| 219 | if (gobj == NULL) { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 220 | ret = -EINVAL; |
| 221 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 222 | } |
| 223 | |
| 224 | p->uf.bo = gem_to_amdgpu_bo(gobj); |
| 225 | p->uf.offset = fence_data->offset; |
| 226 | } else { |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 227 | ret = -EINVAL; |
| 228 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 229 | } |
Christian König | 9a5e8fb | 2015-06-23 17:07:03 +0200 | [diff] [blame] | 230 | break; |
| 231 | |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 232 | case AMDGPU_CHUNK_ID_DEPENDENCIES: |
| 233 | break; |
| 234 | |
Christian König | 9a5e8fb | 2015-06-23 17:07:03 +0200 | [diff] [blame] | 235 | default: |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 236 | ret = -EINVAL; |
| 237 | goto free_partial_kdata; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 238 | } |
| 239 | } |
| 240 | |
monk.liu | e60b344 | 2015-07-17 18:39:25 +0800 | [diff] [blame] | 241 | |
Christian König | b203dd9 | 2015-08-18 18:23:16 +0200 | [diff] [blame] | 242 | p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL); |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 243 | if (!p->ibs) { |
| 244 | ret = -ENOMEM; |
| 245 | goto free_all_kdata; |
| 246 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 247 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 248 | kfree(chunk_array); |
Dan Carpenter | 1d26347 | 2015-09-23 13:59:28 +0300 | [diff] [blame] | 249 | return 0; |
| 250 | |
| 251 | free_all_kdata: |
| 252 | i = p->nchunks - 1; |
| 253 | free_partial_kdata: |
| 254 | for (; i >= 0; i--) |
| 255 | drm_free_large(p->chunks[i].kdata); |
| 256 | kfree(p->chunks); |
| 257 | put_bo_list: |
| 258 | if (p->bo_list) |
| 259 | amdgpu_bo_list_put(p->bo_list); |
| 260 | amdgpu_ctx_put(p->ctx); |
| 261 | free_chunk: |
| 262 | kfree(chunk_array); |
| 263 | |
| 264 | return ret; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 265 | } |
| 266 | |
| 267 | /* Returns how many bytes TTM can move per IB. |
| 268 | */ |
| 269 | static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev) |
| 270 | { |
| 271 | u64 real_vram_size = adev->mc.real_vram_size; |
| 272 | u64 vram_usage = atomic64_read(&adev->vram_usage); |
| 273 | |
| 274 | /* This function is based on the current VRAM usage. |
| 275 | * |
| 276 | * - If all of VRAM is free, allow relocating the number of bytes that |
| 277 | * is equal to 1/4 of the size of VRAM for this IB. |
| 278 | |
| 279 | * - If more than one half of VRAM is occupied, only allow relocating |
| 280 | * 1 MB of data for this IB. |
| 281 | * |
| 282 | * - From 0 to one half of used VRAM, the threshold decreases |
| 283 | * linearly. |
| 284 | * __________________ |
| 285 | * 1/4 of -|\ | |
| 286 | * VRAM | \ | |
| 287 | * | \ | |
| 288 | * | \ | |
| 289 | * | \ | |
| 290 | * | \ | |
| 291 | * | \ | |
| 292 | * | \________|1 MB |
| 293 | * |----------------| |
| 294 | * VRAM 0 % 100 % |
| 295 | * used used |
| 296 | * |
| 297 | * Note: It's a threshold, not a limit. The threshold must be crossed |
| 298 | * for buffer relocations to stop, so any buffer of an arbitrary size |
| 299 | * can be moved as long as the threshold isn't crossed before |
| 300 | * the relocation takes place. We don't want to disable buffer |
| 301 | * relocations completely. |
| 302 | * |
| 303 | * The idea is that buffers should be placed in VRAM at creation time |
| 304 | * and TTM should only do a minimum number of relocations during |
| 305 | * command submission. In practice, you need to submit at least |
| 306 | * a dozen IBs to move all buffers to VRAM if they are in GTT. |
| 307 | * |
| 308 | * Also, things can get pretty crazy under memory pressure and actual |
| 309 | * VRAM usage can change a lot, so playing safe even at 50% does |
| 310 | * consistently increase performance. |
| 311 | */ |
| 312 | |
| 313 | u64 half_vram = real_vram_size >> 1; |
| 314 | u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage; |
| 315 | u64 bytes_moved_threshold = half_free_vram >> 1; |
| 316 | return max(bytes_moved_threshold, 1024*1024ull); |
| 317 | } |
| 318 | |
Christian König | a5b7505 | 2015-09-03 16:40:39 +0200 | [diff] [blame] | 319 | int amdgpu_cs_list_validate(struct amdgpu_device *adev, |
| 320 | struct amdgpu_vm *vm, |
| 321 | struct list_head *validated) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 322 | { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 323 | struct amdgpu_bo_list_entry *lobj; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 324 | struct amdgpu_bo *bo; |
| 325 | u64 bytes_moved = 0, initial_bytes_moved; |
| 326 | u64 bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(adev); |
| 327 | int r; |
| 328 | |
Christian König | a5b7505 | 2015-09-03 16:40:39 +0200 | [diff] [blame] | 329 | list_for_each_entry(lobj, validated, tv.head) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 330 | bo = lobj->robj; |
| 331 | if (!bo->pin_count) { |
| 332 | u32 domain = lobj->prefered_domains; |
| 333 | u32 current_domain = |
| 334 | amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); |
| 335 | |
| 336 | /* Check if this buffer will be moved and don't move it |
| 337 | * if we have moved too many buffers for this IB already. |
| 338 | * |
| 339 | * Note that this allows moving at least one buffer of |
| 340 | * any size, because it doesn't take the current "bo" |
| 341 | * into account. We don't want to disallow buffer moves |
| 342 | * completely. |
| 343 | */ |
Christian König | 270e869 | 2015-09-02 20:25:48 +0200 | [diff] [blame] | 344 | if ((lobj->allowed_domains & current_domain) != 0 && |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 345 | (domain & current_domain) == 0 && /* will be moved */ |
| 346 | bytes_moved > bytes_moved_threshold) { |
| 347 | /* don't move it */ |
| 348 | domain = current_domain; |
| 349 | } |
| 350 | |
| 351 | retry: |
| 352 | amdgpu_ttm_placement_from_domain(bo, domain); |
| 353 | initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); |
| 354 | r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); |
| 355 | bytes_moved += atomic64_read(&adev->num_bytes_moved) - |
| 356 | initial_bytes_moved; |
| 357 | |
| 358 | if (unlikely(r)) { |
| 359 | if (r != -ERESTARTSYS && domain != lobj->allowed_domains) { |
| 360 | domain = lobj->allowed_domains; |
| 361 | goto retry; |
| 362 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 363 | return r; |
| 364 | } |
| 365 | } |
| 366 | lobj->bo_va = amdgpu_vm_bo_find(vm, bo); |
| 367 | } |
| 368 | return 0; |
| 369 | } |
| 370 | |
| 371 | static int amdgpu_cs_parser_relocs(struct amdgpu_cs_parser *p) |
| 372 | { |
| 373 | struct amdgpu_fpriv *fpriv = p->filp->driver_priv; |
| 374 | struct amdgpu_cs_buckets buckets; |
Christian König | a5b7505 | 2015-09-03 16:40:39 +0200 | [diff] [blame] | 375 | struct list_head duplicates; |
monk.liu | 840d514 | 2015-04-27 15:19:20 +0800 | [diff] [blame] | 376 | bool need_mmap_lock = false; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 377 | int i, r; |
| 378 | |
monk.liu | 840d514 | 2015-04-27 15:19:20 +0800 | [diff] [blame] | 379 | if (p->bo_list) { |
| 380 | need_mmap_lock = p->bo_list->has_userptr; |
| 381 | amdgpu_cs_buckets_init(&buckets); |
| 382 | for (i = 0; i < p->bo_list->num_entries; i++) |
| 383 | amdgpu_cs_buckets_add(&buckets, &p->bo_list->array[i].tv.head, |
| 384 | p->bo_list->array[i].priority); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 385 | |
monk.liu | 840d514 | 2015-04-27 15:19:20 +0800 | [diff] [blame] | 386 | amdgpu_cs_buckets_get_list(&buckets, &p->validated); |
| 387 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 388 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 389 | p->vm_bos = amdgpu_vm_get_bos(p->adev, &fpriv->vm, |
| 390 | &p->validated); |
| 391 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 392 | if (need_mmap_lock) |
| 393 | down_read(¤t->mm->mmap_sem); |
| 394 | |
Christian König | a5b7505 | 2015-09-03 16:40:39 +0200 | [diff] [blame] | 395 | INIT_LIST_HEAD(&duplicates); |
| 396 | r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); |
| 397 | if (unlikely(r != 0)) |
| 398 | goto error_reserve; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 399 | |
Christian König | a5b7505 | 2015-09-03 16:40:39 +0200 | [diff] [blame] | 400 | r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &p->validated); |
| 401 | if (r) |
| 402 | goto error_validate; |
| 403 | |
| 404 | r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &duplicates); |
| 405 | |
| 406 | error_validate: |
| 407 | if (r) |
| 408 | ttm_eu_backoff_reservation(&p->ticket, &p->validated); |
| 409 | |
| 410 | error_reserve: |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 411 | if (need_mmap_lock) |
| 412 | up_read(¤t->mm->mmap_sem); |
| 413 | |
| 414 | return r; |
| 415 | } |
| 416 | |
| 417 | static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) |
| 418 | { |
| 419 | struct amdgpu_bo_list_entry *e; |
| 420 | int r; |
| 421 | |
| 422 | list_for_each_entry(e, &p->validated, tv.head) { |
| 423 | struct reservation_object *resv = e->robj->tbo.resv; |
| 424 | r = amdgpu_sync_resv(p->adev, &p->ibs[0].sync, resv, p->filp); |
| 425 | |
| 426 | if (r) |
| 427 | return r; |
| 428 | } |
| 429 | return 0; |
| 430 | } |
| 431 | |
| 432 | static int cmp_size_smaller_first(void *priv, struct list_head *a, |
| 433 | struct list_head *b) |
| 434 | { |
| 435 | struct amdgpu_bo_list_entry *la = list_entry(a, struct amdgpu_bo_list_entry, tv.head); |
| 436 | struct amdgpu_bo_list_entry *lb = list_entry(b, struct amdgpu_bo_list_entry, tv.head); |
| 437 | |
| 438 | /* Sort A before B if A is smaller. */ |
| 439 | return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages; |
| 440 | } |
| 441 | |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 442 | /** |
| 443 | * cs_parser_fini() - clean parser states |
| 444 | * @parser: parser structure holding parsing context. |
| 445 | * @error: error number |
| 446 | * |
| 447 | * If error is set than unvalidate buffer, otherwise just free memory |
| 448 | * used by parsing context. |
| 449 | **/ |
| 450 | static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff) |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 451 | { |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 452 | unsigned i; |
| 453 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 454 | if (!error) { |
| 455 | /* Sort the buffer list from the smallest to largest buffer, |
| 456 | * which affects the order of buffers in the LRU list. |
| 457 | * This assures that the smallest buffers are added first |
| 458 | * to the LRU list, so they are likely to be later evicted |
| 459 | * first, instead of large buffers whose eviction is more |
| 460 | * expensive. |
| 461 | * |
| 462 | * This slightly lowers the number of bytes moved by TTM |
| 463 | * per frame under memory pressure. |
| 464 | */ |
| 465 | list_sort(NULL, &parser->validated, cmp_size_smaller_first); |
| 466 | |
| 467 | ttm_eu_fence_buffer_objects(&parser->ticket, |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 468 | &parser->validated, |
| 469 | parser->fence); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 470 | } else if (backoff) { |
| 471 | ttm_eu_backoff_reservation(&parser->ticket, |
| 472 | &parser->validated); |
| 473 | } |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 474 | fence_put(parser->fence); |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 475 | |
Christian König | 3cb485f | 2015-05-11 15:34:59 +0200 | [diff] [blame] | 476 | if (parser->ctx) |
| 477 | amdgpu_ctx_put(parser->ctx); |
Chunming Zhou | a3348bb | 2015-08-18 16:25:46 +0800 | [diff] [blame] | 478 | if (parser->bo_list) |
| 479 | amdgpu_bo_list_put(parser->bo_list); |
| 480 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 481 | drm_free_large(parser->vm_bos); |
| 482 | for (i = 0; i < parser->nchunks; i++) |
| 483 | drm_free_large(parser->chunks[i].kdata); |
| 484 | kfree(parser->chunks); |
Christian König | e4a58a2 | 2015-11-05 17:00:25 +0100 | [diff] [blame] | 485 | if (parser->ibs) |
| 486 | for (i = 0; i < parser->num_ibs; i++) |
| 487 | amdgpu_ib_free(parser->adev, &parser->ibs[i]); |
| 488 | kfree(parser->ibs); |
| 489 | if (parser->uf.bo) |
| 490 | drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 491 | } |
| 492 | |
| 493 | static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p, |
| 494 | struct amdgpu_vm *vm) |
| 495 | { |
| 496 | struct amdgpu_device *adev = p->adev; |
| 497 | struct amdgpu_bo_va *bo_va; |
| 498 | struct amdgpu_bo *bo; |
| 499 | int i, r; |
| 500 | |
| 501 | r = amdgpu_vm_update_page_directory(adev, vm); |
| 502 | if (r) |
| 503 | return r; |
| 504 | |
Bas Nieuwenhuizen | 05906de | 2015-08-14 20:08:40 +0200 | [diff] [blame] | 505 | r = amdgpu_sync_fence(adev, &p->ibs[0].sync, vm->page_directory_fence); |
| 506 | if (r) |
| 507 | return r; |
| 508 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 509 | r = amdgpu_vm_clear_freed(adev, vm); |
| 510 | if (r) |
| 511 | return r; |
| 512 | |
| 513 | if (p->bo_list) { |
| 514 | for (i = 0; i < p->bo_list->num_entries; i++) { |
Christian König | 91e1a52 | 2015-07-06 22:06:40 +0200 | [diff] [blame] | 515 | struct fence *f; |
| 516 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 517 | /* ignore duplicates */ |
| 518 | bo = p->bo_list->array[i].robj; |
| 519 | if (!bo) |
| 520 | continue; |
| 521 | |
| 522 | bo_va = p->bo_list->array[i].bo_va; |
| 523 | if (bo_va == NULL) |
| 524 | continue; |
| 525 | |
| 526 | r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem); |
| 527 | if (r) |
| 528 | return r; |
| 529 | |
Chunming Zhou | bb1e38a4 | 2015-08-03 18:19:38 +0800 | [diff] [blame] | 530 | f = bo_va->last_pt_update; |
Christian König | 91e1a52 | 2015-07-06 22:06:40 +0200 | [diff] [blame] | 531 | r = amdgpu_sync_fence(adev, &p->ibs[0].sync, f); |
| 532 | if (r) |
| 533 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 534 | } |
Christian König | b495bd3 | 2015-09-10 14:00:35 +0200 | [diff] [blame] | 535 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 536 | } |
| 537 | |
Christian König | b495bd3 | 2015-09-10 14:00:35 +0200 | [diff] [blame] | 538 | r = amdgpu_vm_clear_invalids(adev, vm, &p->ibs[0].sync); |
| 539 | |
| 540 | if (amdgpu_vm_debug && p->bo_list) { |
| 541 | /* Invalidate all BOs to test for userspace bugs */ |
| 542 | for (i = 0; i < p->bo_list->num_entries; i++) { |
| 543 | /* ignore duplicates */ |
| 544 | bo = p->bo_list->array[i].robj; |
| 545 | if (!bo) |
| 546 | continue; |
| 547 | |
| 548 | amdgpu_vm_bo_invalidate(adev, bo); |
| 549 | } |
| 550 | } |
| 551 | |
| 552 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 553 | } |
| 554 | |
| 555 | static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, |
| 556 | struct amdgpu_cs_parser *parser) |
| 557 | { |
| 558 | struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; |
| 559 | struct amdgpu_vm *vm = &fpriv->vm; |
| 560 | struct amdgpu_ring *ring; |
| 561 | int i, r; |
| 562 | |
| 563 | if (parser->num_ibs == 0) |
| 564 | return 0; |
| 565 | |
| 566 | /* Only for UVD/VCE VM emulation */ |
| 567 | for (i = 0; i < parser->num_ibs; i++) { |
| 568 | ring = parser->ibs[i].ring; |
| 569 | if (ring->funcs->parse_cs) { |
| 570 | r = amdgpu_ring_parse_cs(ring, parser, i); |
| 571 | if (r) |
| 572 | return r; |
| 573 | } |
| 574 | } |
| 575 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 576 | r = amdgpu_bo_vm_update_pte(parser, vm); |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 577 | if (!r) |
| 578 | amdgpu_cs_sync_rings(parser); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 579 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 580 | return r; |
| 581 | } |
| 582 | |
| 583 | static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r) |
| 584 | { |
| 585 | if (r == -EDEADLK) { |
| 586 | r = amdgpu_gpu_reset(adev); |
| 587 | if (!r) |
| 588 | r = -EAGAIN; |
| 589 | } |
| 590 | return r; |
| 591 | } |
| 592 | |
| 593 | static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, |
| 594 | struct amdgpu_cs_parser *parser) |
| 595 | { |
| 596 | struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; |
| 597 | struct amdgpu_vm *vm = &fpriv->vm; |
| 598 | int i, j; |
| 599 | int r; |
| 600 | |
| 601 | for (i = 0, j = 0; i < parser->nchunks && j < parser->num_ibs; i++) { |
| 602 | struct amdgpu_cs_chunk *chunk; |
| 603 | struct amdgpu_ib *ib; |
| 604 | struct drm_amdgpu_cs_chunk_ib *chunk_ib; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 605 | struct amdgpu_ring *ring; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 606 | |
| 607 | chunk = &parser->chunks[i]; |
| 608 | ib = &parser->ibs[j]; |
| 609 | chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; |
| 610 | |
| 611 | if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) |
| 612 | continue; |
| 613 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 614 | r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type, |
| 615 | chunk_ib->ip_instance, chunk_ib->ring, |
| 616 | &ring); |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 617 | if (r) |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 618 | return r; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 619 | |
| 620 | if (ring->funcs->parse_cs) { |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 621 | struct amdgpu_bo_va_mapping *m; |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 622 | struct amdgpu_bo *aobj = NULL; |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 623 | uint64_t offset; |
| 624 | uint8_t *kptr; |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 625 | |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 626 | m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start, |
| 627 | &aobj); |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 628 | if (!aobj) { |
| 629 | DRM_ERROR("IB va_start is invalid\n"); |
| 630 | return -EINVAL; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 631 | } |
| 632 | |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 633 | if ((chunk_ib->va_start + chunk_ib->ib_bytes) > |
| 634 | (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) { |
| 635 | DRM_ERROR("IB va_start+ib_bytes is invalid\n"); |
| 636 | return -EINVAL; |
| 637 | } |
| 638 | |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 639 | /* the IB should be reserved at this point */ |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 640 | r = amdgpu_bo_kmap(aobj, (void **)&kptr); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 641 | if (r) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 642 | return r; |
| 643 | } |
| 644 | |
Christian König | 4802ce1 | 2015-06-10 17:20:11 +0200 | [diff] [blame] | 645 | offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE; |
| 646 | kptr += chunk_ib->va_start - offset; |
| 647 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 648 | r = amdgpu_ib_get(ring, NULL, chunk_ib->ib_bytes, ib); |
| 649 | if (r) { |
| 650 | DRM_ERROR("Failed to get ib !\n"); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 651 | return r; |
| 652 | } |
| 653 | |
| 654 | memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); |
| 655 | amdgpu_bo_kunmap(aobj); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 656 | } else { |
| 657 | r = amdgpu_ib_get(ring, vm, 0, ib); |
| 658 | if (r) { |
| 659 | DRM_ERROR("Failed to get ib !\n"); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 660 | return r; |
| 661 | } |
| 662 | |
| 663 | ib->gpu_addr = chunk_ib->va_start; |
| 664 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 665 | |
Marek Olšák | 3ccec53 | 2015-06-02 17:44:49 +0200 | [diff] [blame] | 666 | ib->length_dw = chunk_ib->ib_bytes / 4; |
Jammy Zhou | de807f8 | 2015-05-11 23:41:41 +0800 | [diff] [blame] | 667 | ib->flags = chunk_ib->flags; |
Christian König | 3cb485f | 2015-05-11 15:34:59 +0200 | [diff] [blame] | 668 | ib->ctx = parser->ctx; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 669 | j++; |
| 670 | } |
| 671 | |
| 672 | if (!parser->num_ibs) |
| 673 | return 0; |
| 674 | |
| 675 | /* add GDS resources to first IB */ |
| 676 | if (parser->bo_list) { |
| 677 | struct amdgpu_bo *gds = parser->bo_list->gds_obj; |
| 678 | struct amdgpu_bo *gws = parser->bo_list->gws_obj; |
| 679 | struct amdgpu_bo *oa = parser->bo_list->oa_obj; |
| 680 | struct amdgpu_ib *ib = &parser->ibs[0]; |
| 681 | |
| 682 | if (gds) { |
| 683 | ib->gds_base = amdgpu_bo_gpu_offset(gds); |
| 684 | ib->gds_size = amdgpu_bo_size(gds); |
| 685 | } |
| 686 | if (gws) { |
| 687 | ib->gws_base = amdgpu_bo_gpu_offset(gws); |
| 688 | ib->gws_size = amdgpu_bo_size(gws); |
| 689 | } |
| 690 | if (oa) { |
| 691 | ib->oa_base = amdgpu_bo_gpu_offset(oa); |
| 692 | ib->oa_size = amdgpu_bo_size(oa); |
| 693 | } |
| 694 | } |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 695 | /* wrap the last IB with user fence */ |
| 696 | if (parser->uf.bo) { |
| 697 | struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1]; |
| 698 | |
| 699 | /* UVD & VCE fw doesn't support user fences */ |
| 700 | if (ib->ring->type == AMDGPU_RING_TYPE_UVD || |
| 701 | ib->ring->type == AMDGPU_RING_TYPE_VCE) |
| 702 | return -EINVAL; |
| 703 | |
| 704 | ib->user = &parser->uf; |
| 705 | } |
| 706 | |
| 707 | return 0; |
| 708 | } |
| 709 | |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 710 | static int amdgpu_cs_dependencies(struct amdgpu_device *adev, |
| 711 | struct amdgpu_cs_parser *p) |
| 712 | { |
Christian König | 76a1ea6 | 2015-07-06 19:42:10 +0200 | [diff] [blame] | 713 | struct amdgpu_fpriv *fpriv = p->filp->driver_priv; |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 714 | struct amdgpu_ib *ib; |
| 715 | int i, j, r; |
| 716 | |
| 717 | if (!p->num_ibs) |
| 718 | return 0; |
| 719 | |
| 720 | /* Add dependencies to first IB */ |
| 721 | ib = &p->ibs[0]; |
| 722 | for (i = 0; i < p->nchunks; ++i) { |
| 723 | struct drm_amdgpu_cs_chunk_dep *deps; |
| 724 | struct amdgpu_cs_chunk *chunk; |
| 725 | unsigned num_deps; |
| 726 | |
| 727 | chunk = &p->chunks[i]; |
| 728 | |
| 729 | if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES) |
| 730 | continue; |
| 731 | |
| 732 | deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; |
| 733 | num_deps = chunk->length_dw * 4 / |
| 734 | sizeof(struct drm_amdgpu_cs_chunk_dep); |
| 735 | |
| 736 | for (j = 0; j < num_deps; ++j) { |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 737 | struct amdgpu_ring *ring; |
Christian König | 76a1ea6 | 2015-07-06 19:42:10 +0200 | [diff] [blame] | 738 | struct amdgpu_ctx *ctx; |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 739 | struct fence *fence; |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 740 | |
| 741 | r = amdgpu_cs_get_ring(adev, deps[j].ip_type, |
| 742 | deps[j].ip_instance, |
| 743 | deps[j].ring, &ring); |
| 744 | if (r) |
| 745 | return r; |
| 746 | |
Christian König | 76a1ea6 | 2015-07-06 19:42:10 +0200 | [diff] [blame] | 747 | ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id); |
| 748 | if (ctx == NULL) |
| 749 | return -EINVAL; |
| 750 | |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 751 | fence = amdgpu_ctx_get_fence(ctx, ring, |
| 752 | deps[j].handle); |
| 753 | if (IS_ERR(fence)) { |
| 754 | r = PTR_ERR(fence); |
Christian König | 76a1ea6 | 2015-07-06 19:42:10 +0200 | [diff] [blame] | 755 | amdgpu_ctx_put(ctx); |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 756 | return r; |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 757 | |
| 758 | } else if (fence) { |
| 759 | r = amdgpu_sync_fence(adev, &ib->sync, fence); |
| 760 | fence_put(fence); |
| 761 | amdgpu_ctx_put(ctx); |
| 762 | if (r) |
| 763 | return r; |
Christian König | 76a1ea6 | 2015-07-06 19:42:10 +0200 | [diff] [blame] | 764 | } |
Christian König | 2b48d32 | 2015-06-19 17:31:29 +0200 | [diff] [blame] | 765 | } |
| 766 | } |
| 767 | |
| 768 | return 0; |
| 769 | } |
| 770 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 771 | static int amdgpu_cs_free_job(struct amdgpu_job *job) |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 772 | { |
| 773 | int i; |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 774 | if (job->ibs) |
| 775 | for (i = 0; i < job->num_ibs; i++) |
| 776 | amdgpu_ib_free(job->adev, &job->ibs[i]); |
| 777 | kfree(job->ibs); |
| 778 | if (job->uf.bo) |
| 779 | drm_gem_object_unreference_unlocked(&job->uf.bo->gem_base); |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 780 | return 0; |
| 781 | } |
| 782 | |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 783 | int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) |
| 784 | { |
| 785 | struct amdgpu_device *adev = dev->dev_private; |
| 786 | union drm_amdgpu_cs *cs = data; |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 787 | struct amdgpu_cs_parser parser = {}; |
Christian König | 26a6980 | 2015-08-18 21:09:33 +0200 | [diff] [blame] | 788 | bool reserved_buffers = false; |
| 789 | int i, r; |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 790 | |
Christian König | 0c418f1 | 2015-09-01 15:13:53 +0200 | [diff] [blame] | 791 | if (!adev->accel_working) |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 792 | return -EBUSY; |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 793 | |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 794 | parser.adev = adev; |
| 795 | parser.filp = filp; |
| 796 | |
| 797 | r = amdgpu_cs_parser_init(&parser, data); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 798 | if (r) { |
Chunming Zhou | 049fc52 | 2015-07-21 14:36:51 +0800 | [diff] [blame] | 799 | DRM_ERROR("Failed to initialize parser !\n"); |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 800 | amdgpu_cs_parser_fini(&parser, r, false); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 801 | r = amdgpu_cs_handle_lockup(adev, r); |
| 802 | return r; |
| 803 | } |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 804 | r = amdgpu_cs_parser_relocs(&parser); |
Christian König | 26a6980 | 2015-08-18 21:09:33 +0200 | [diff] [blame] | 805 | if (r == -ENOMEM) |
| 806 | DRM_ERROR("Not enough memory for command submission!\n"); |
| 807 | else if (r && r != -ERESTARTSYS) |
| 808 | DRM_ERROR("Failed to process the buffer list %d!\n", r); |
| 809 | else if (!r) { |
| 810 | reserved_buffers = true; |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 811 | r = amdgpu_cs_ib_fill(adev, &parser); |
Christian König | 26a6980 | 2015-08-18 21:09:33 +0200 | [diff] [blame] | 812 | } |
| 813 | |
| 814 | if (!r) { |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 815 | r = amdgpu_cs_dependencies(adev, &parser); |
Christian König | 26a6980 | 2015-08-18 21:09:33 +0200 | [diff] [blame] | 816 | if (r) |
| 817 | DRM_ERROR("Failed in the dependencies handling %d!\n", r); |
| 818 | } |
| 819 | |
| 820 | if (r) |
| 821 | goto out; |
| 822 | |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 823 | for (i = 0; i < parser.num_ibs; i++) |
| 824 | trace_amdgpu_cs(&parser, i); |
Christian König | 26a6980 | 2015-08-18 21:09:33 +0200 | [diff] [blame] | 825 | |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 826 | r = amdgpu_cs_ib_vm_chunk(adev, &parser); |
Chunming Zhou | 4fe6311 | 2015-08-18 16:12:15 +0800 | [diff] [blame] | 827 | if (r) |
| 828 | goto out; |
| 829 | |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 830 | if (amdgpu_enable_scheduler && parser.num_ibs) { |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 831 | struct amdgpu_ring * ring = parser.ibs->ring; |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 832 | struct amd_sched_fence *fence; |
| 833 | struct amdgpu_job *job; |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 834 | |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 835 | job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL); |
Dan Carpenter | 4cfdcd9 | 2015-11-04 16:25:09 +0300 | [diff] [blame] | 836 | if (!job) { |
| 837 | r = -ENOMEM; |
| 838 | goto out; |
| 839 | } |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 840 | |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 841 | job->base.sched = &ring->sched; |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 842 | job->base.s_entity = &parser.ctx->rings[ring->idx].entity; |
| 843 | job->adev = parser.adev; |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 844 | job->owner = parser.filp; |
| 845 | job->free_job = amdgpu_cs_free_job; |
| 846 | |
Christian König | 5d82730 | 2015-11-13 13:04:50 +0100 | [diff] [blame] | 847 | job->ibs = parser.ibs; |
| 848 | job->num_ibs = parser.num_ibs; |
| 849 | parser.ibs = NULL; |
| 850 | parser.num_ibs = 0; |
| 851 | |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 852 | if (job->ibs[job->num_ibs - 1].user) { |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 853 | job->uf = parser.uf; |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 854 | job->ibs[job->num_ibs - 1].user = &job->uf; |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 855 | parser.uf.bo = NULL; |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 856 | } |
| 857 | |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 858 | fence = amd_sched_fence_create(job->base.s_entity, |
| 859 | parser.filp); |
| 860 | if (!fence) { |
| 861 | r = -ENOMEM; |
Chunming Zhou | bb977d3 | 2015-08-18 15:16:40 +0800 | [diff] [blame] | 862 | amdgpu_cs_free_job(job); |
| 863 | kfree(job); |
Chunming Zhou | f556cb0c | 2015-08-02 11:18:04 +0800 | [diff] [blame] | 864 | goto out; |
| 865 | } |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 866 | job->base.s_fence = fence; |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 867 | parser.fence = fence_get(&fence->base); |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 868 | |
| 869 | cs->out.handle = amdgpu_ctx_add_fence(parser.ctx, ring, |
| 870 | &fence->base); |
Christian König | e4a58a2 | 2015-11-05 17:00:25 +0100 | [diff] [blame] | 871 | job->ibs[job->num_ibs - 1].sequence = cs->out.handle; |
Christian König | eb98d1c | 2015-08-20 17:28:36 +0200 | [diff] [blame] | 872 | |
Chunming Zhou | 7034dec | 2015-11-11 14:56:00 +0800 | [diff] [blame] | 873 | trace_amdgpu_cs_ioctl(job); |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 874 | amd_sched_entity_push_job(&job->base); |
| 875 | |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 876 | } else { |
| 877 | struct amdgpu_fence *fence; |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 878 | |
Christian König | 984810f | 2015-11-14 21:05:35 +0100 | [diff] [blame] | 879 | r = amdgpu_ib_schedule(adev, parser.num_ibs, parser.ibs, |
| 880 | parser.filp); |
| 881 | fence = parser.ibs[parser.num_ibs - 1].fence; |
| 882 | parser.fence = fence_get(&fence->base); |
| 883 | cs->out.handle = parser.ibs[parser.num_ibs - 1].sequence; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 884 | } |
| 885 | |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 886 | out: |
Christian König | 7e52a81 | 2015-11-04 15:44:39 +0100 | [diff] [blame] | 887 | amdgpu_cs_parser_fini(&parser, r, reserved_buffers); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 888 | r = amdgpu_cs_handle_lockup(adev, r); |
| 889 | return r; |
| 890 | } |
| 891 | |
| 892 | /** |
| 893 | * amdgpu_cs_wait_ioctl - wait for a command submission to finish |
| 894 | * |
| 895 | * @dev: drm device |
| 896 | * @data: data from userspace |
| 897 | * @filp: file private |
| 898 | * |
| 899 | * Wait for the command submission identified by handle to finish. |
| 900 | */ |
| 901 | int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, |
| 902 | struct drm_file *filp) |
| 903 | { |
| 904 | union drm_amdgpu_wait_cs *wait = data; |
| 905 | struct amdgpu_device *adev = dev->dev_private; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 906 | unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); |
Christian König | 03507c4 | 2015-06-19 17:00:19 +0200 | [diff] [blame] | 907 | struct amdgpu_ring *ring = NULL; |
Jammy Zhou | 66b3cf2 | 2015-05-08 17:29:40 +0800 | [diff] [blame] | 908 | struct amdgpu_ctx *ctx; |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 909 | struct fence *fence; |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 910 | long r; |
| 911 | |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 912 | r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance, |
| 913 | wait->in.ring, &ring); |
| 914 | if (r) |
| 915 | return r; |
| 916 | |
Jammy Zhou | 66b3cf2 | 2015-05-08 17:29:40 +0800 | [diff] [blame] | 917 | ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); |
| 918 | if (ctx == NULL) |
| 919 | return -EINVAL; |
Chunming Zhou | 4b559c9 | 2015-07-21 15:53:04 +0800 | [diff] [blame] | 920 | |
| 921 | fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); |
| 922 | if (IS_ERR(fence)) |
| 923 | r = PTR_ERR(fence); |
| 924 | else if (fence) { |
| 925 | r = fence_wait_timeout(fence, true, timeout); |
| 926 | fence_put(fence); |
| 927 | } else |
Christian König | 21c16bf | 2015-07-07 17:24:49 +0200 | [diff] [blame] | 928 | r = 1; |
| 929 | |
Jammy Zhou | 66b3cf2 | 2015-05-08 17:29:40 +0800 | [diff] [blame] | 930 | amdgpu_ctx_put(ctx); |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 931 | if (r < 0) |
| 932 | return r; |
| 933 | |
| 934 | memset(wait, 0, sizeof(*wait)); |
| 935 | wait->out.status = (r == 0); |
| 936 | |
| 937 | return 0; |
| 938 | } |
| 939 | |
| 940 | /** |
| 941 | * amdgpu_cs_find_bo_va - find bo_va for VM address |
| 942 | * |
| 943 | * @parser: command submission parser context |
| 944 | * @addr: VM address |
| 945 | * @bo: resulting BO of the mapping found |
| 946 | * |
| 947 | * Search the buffer objects in the command submission context for a certain |
| 948 | * virtual memory address. Returns allocation structure when found, NULL |
| 949 | * otherwise. |
| 950 | */ |
| 951 | struct amdgpu_bo_va_mapping * |
| 952 | amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, |
| 953 | uint64_t addr, struct amdgpu_bo **bo) |
| 954 | { |
| 955 | struct amdgpu_bo_list_entry *reloc; |
| 956 | struct amdgpu_bo_va_mapping *mapping; |
| 957 | |
| 958 | addr /= AMDGPU_GPU_PAGE_SIZE; |
| 959 | |
| 960 | list_for_each_entry(reloc, &parser->validated, tv.head) { |
| 961 | if (!reloc->bo_va) |
| 962 | continue; |
| 963 | |
Christian König | 7fc1195 | 2015-07-30 11:53:42 +0200 | [diff] [blame] | 964 | list_for_each_entry(mapping, &reloc->bo_va->valids, list) { |
| 965 | if (mapping->it.start > addr || |
| 966 | addr > mapping->it.last) |
| 967 | continue; |
| 968 | |
| 969 | *bo = reloc->bo_va->bo; |
| 970 | return mapping; |
| 971 | } |
| 972 | |
| 973 | list_for_each_entry(mapping, &reloc->bo_va->invalids, list) { |
Alex Deucher | d38ceaf | 2015-04-20 16:55:21 -0400 | [diff] [blame] | 974 | if (mapping->it.start > addr || |
| 975 | addr > mapping->it.last) |
| 976 | continue; |
| 977 | |
| 978 | *bo = reloc->bo_va->bo; |
| 979 | return mapping; |
| 980 | } |
| 981 | } |
| 982 | |
| 983 | return NULL; |
| 984 | } |