Jerome Glisse | 771fe6b | 2009-06-05 14:42:42 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2009 Jerome Glisse. |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 6 | * copy of this software and associated documentation files (the |
| 7 | * "Software"), to deal in the Software without restriction, including |
| 8 | * without limitation the rights to use, copy, modify, merge, publish, |
| 9 | * distribute, sub license, and/or sell copies of the Software, and to |
| 10 | * permit persons to whom the Software is furnished to do so, subject to |
| 11 | * the following conditions: |
| 12 | * |
| 13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
| 16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
| 17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| 18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
| 19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 20 | * |
| 21 | * The above copyright notice and this permission notice (including the |
| 22 | * next paragraph) shall be included in all copies or substantial portions |
| 23 | * of the Software. |
| 24 | * |
| 25 | */ |
| 26 | /* |
| 27 | * Authors: |
| 28 | * Jerome Glisse <glisse@freedesktop.org> |
| 29 | * Dave Airlie |
| 30 | */ |
| 31 | #include <linux/seq_file.h> |
| 32 | #include <asm/atomic.h> |
| 33 | #include <linux/wait.h> |
| 34 | #include <linux/list.h> |
| 35 | #include <linux/kref.h> |
| 36 | #include "drmP.h" |
| 37 | #include "drm.h" |
| 38 | #include "radeon_reg.h" |
| 39 | #include "radeon.h" |
| 40 | |
| 41 | int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) |
| 42 | { |
| 43 | unsigned long irq_flags; |
| 44 | |
| 45 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 46 | if (fence->emited) { |
| 47 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 48 | return 0; |
| 49 | } |
| 50 | fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); |
| 51 | if (!rdev->cp.ready) { |
| 52 | /* FIXME: cp is not running assume everythings is done right |
| 53 | * away |
| 54 | */ |
| 55 | WREG32(rdev->fence_drv.scratch_reg, fence->seq); |
| 56 | } else { |
| 57 | radeon_fence_ring_emit(rdev, fence); |
| 58 | } |
| 59 | fence->emited = true; |
| 60 | fence->timeout = jiffies + ((2000 * HZ) / 1000); |
| 61 | list_del(&fence->list); |
| 62 | list_add_tail(&fence->list, &rdev->fence_drv.emited); |
| 63 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 64 | return 0; |
| 65 | } |
| 66 | |
| 67 | static bool radeon_fence_poll_locked(struct radeon_device *rdev) |
| 68 | { |
| 69 | struct radeon_fence *fence; |
| 70 | struct list_head *i, *n; |
| 71 | uint32_t seq; |
| 72 | bool wake = false; |
| 73 | |
| 74 | if (rdev == NULL) { |
| 75 | return true; |
| 76 | } |
| 77 | if (rdev->shutdown) { |
| 78 | return true; |
| 79 | } |
| 80 | seq = RREG32(rdev->fence_drv.scratch_reg); |
| 81 | rdev->fence_drv.last_seq = seq; |
| 82 | n = NULL; |
| 83 | list_for_each(i, &rdev->fence_drv.emited) { |
| 84 | fence = list_entry(i, struct radeon_fence, list); |
| 85 | if (fence->seq == seq) { |
| 86 | n = i; |
| 87 | break; |
| 88 | } |
| 89 | } |
| 90 | /* all fence previous to this one are considered as signaled */ |
| 91 | if (n) { |
| 92 | i = n; |
| 93 | do { |
| 94 | n = i->prev; |
| 95 | list_del(i); |
| 96 | list_add_tail(i, &rdev->fence_drv.signaled); |
| 97 | fence = list_entry(i, struct radeon_fence, list); |
| 98 | fence->signaled = true; |
| 99 | i = n; |
| 100 | } while (i != &rdev->fence_drv.emited); |
| 101 | wake = true; |
| 102 | } |
| 103 | return wake; |
| 104 | } |
| 105 | |
| 106 | static void radeon_fence_destroy(struct kref *kref) |
| 107 | { |
| 108 | unsigned long irq_flags; |
| 109 | struct radeon_fence *fence; |
| 110 | |
| 111 | fence = container_of(kref, struct radeon_fence, kref); |
| 112 | write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); |
| 113 | list_del(&fence->list); |
| 114 | fence->emited = false; |
| 115 | write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); |
| 116 | kfree(fence); |
| 117 | } |
| 118 | |
| 119 | int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) |
| 120 | { |
| 121 | unsigned long irq_flags; |
| 122 | |
| 123 | *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); |
| 124 | if ((*fence) == NULL) { |
| 125 | return -ENOMEM; |
| 126 | } |
| 127 | kref_init(&((*fence)->kref)); |
| 128 | (*fence)->rdev = rdev; |
| 129 | (*fence)->emited = false; |
| 130 | (*fence)->signaled = false; |
| 131 | (*fence)->seq = 0; |
| 132 | INIT_LIST_HEAD(&(*fence)->list); |
| 133 | |
| 134 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 135 | list_add_tail(&(*fence)->list, &rdev->fence_drv.created); |
| 136 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 137 | return 0; |
| 138 | } |
| 139 | |
| 140 | |
| 141 | bool radeon_fence_signaled(struct radeon_fence *fence) |
| 142 | { |
| 143 | struct radeon_device *rdev = fence->rdev; |
| 144 | unsigned long irq_flags; |
| 145 | bool signaled = false; |
| 146 | |
| 147 | if (rdev->gpu_lockup) { |
| 148 | return true; |
| 149 | } |
| 150 | if (fence == NULL) { |
| 151 | return true; |
| 152 | } |
| 153 | write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); |
| 154 | signaled = fence->signaled; |
| 155 | /* if we are shuting down report all fence as signaled */ |
| 156 | if (fence->rdev->shutdown) { |
| 157 | signaled = true; |
| 158 | } |
| 159 | if (!fence->emited) { |
| 160 | WARN(1, "Querying an unemited fence : %p !\n", fence); |
| 161 | signaled = true; |
| 162 | } |
| 163 | if (!signaled) { |
| 164 | radeon_fence_poll_locked(fence->rdev); |
| 165 | signaled = fence->signaled; |
| 166 | } |
| 167 | write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); |
| 168 | return signaled; |
| 169 | } |
| 170 | |
| 171 | int radeon_fence_wait(struct radeon_fence *fence, bool interruptible) |
| 172 | { |
| 173 | struct radeon_device *rdev; |
| 174 | unsigned long cur_jiffies; |
| 175 | unsigned long timeout; |
| 176 | bool expired = false; |
| 177 | int r; |
| 178 | |
| 179 | |
| 180 | if (fence == NULL) { |
| 181 | WARN(1, "Querying an invalid fence : %p !\n", fence); |
| 182 | return 0; |
| 183 | } |
| 184 | rdev = fence->rdev; |
| 185 | if (radeon_fence_signaled(fence)) { |
| 186 | return 0; |
| 187 | } |
| 188 | retry: |
| 189 | cur_jiffies = jiffies; |
| 190 | timeout = HZ / 100; |
| 191 | if (time_after(fence->timeout, cur_jiffies)) { |
| 192 | timeout = fence->timeout - cur_jiffies; |
| 193 | } |
| 194 | if (interruptible) { |
| 195 | r = wait_event_interruptible_timeout(rdev->fence_drv.queue, |
| 196 | radeon_fence_signaled(fence), timeout); |
| 197 | if (unlikely(r == -ERESTARTSYS)) { |
| 198 | return -ERESTART; |
| 199 | } |
| 200 | } else { |
| 201 | r = wait_event_timeout(rdev->fence_drv.queue, |
| 202 | radeon_fence_signaled(fence), timeout); |
| 203 | } |
| 204 | if (unlikely(!radeon_fence_signaled(fence))) { |
| 205 | if (unlikely(r == 0)) { |
| 206 | expired = true; |
| 207 | } |
| 208 | if (unlikely(expired)) { |
| 209 | timeout = 1; |
| 210 | if (time_after(cur_jiffies, fence->timeout)) { |
| 211 | timeout = cur_jiffies - fence->timeout; |
| 212 | } |
| 213 | timeout = jiffies_to_msecs(timeout); |
| 214 | if (timeout > 500) { |
| 215 | DRM_ERROR("fence(%p:0x%08X) %lums timeout " |
| 216 | "going to reset GPU\n", |
| 217 | fence, fence->seq, timeout); |
| 218 | radeon_gpu_reset(rdev); |
| 219 | WREG32(rdev->fence_drv.scratch_reg, fence->seq); |
| 220 | } |
| 221 | } |
| 222 | goto retry; |
| 223 | } |
| 224 | if (unlikely(expired)) { |
| 225 | rdev->fence_drv.count_timeout++; |
| 226 | cur_jiffies = jiffies; |
| 227 | timeout = 1; |
| 228 | if (time_after(cur_jiffies, fence->timeout)) { |
| 229 | timeout = cur_jiffies - fence->timeout; |
| 230 | } |
| 231 | timeout = jiffies_to_msecs(timeout); |
| 232 | DRM_ERROR("fence(%p:0x%08X) %lums timeout\n", |
| 233 | fence, fence->seq, timeout); |
| 234 | DRM_ERROR("last signaled fence(0x%08X)\n", |
| 235 | rdev->fence_drv.last_seq); |
| 236 | } |
| 237 | return 0; |
| 238 | } |
| 239 | |
| 240 | int radeon_fence_wait_next(struct radeon_device *rdev) |
| 241 | { |
| 242 | unsigned long irq_flags; |
| 243 | struct radeon_fence *fence; |
| 244 | int r; |
| 245 | |
| 246 | if (rdev->gpu_lockup) { |
| 247 | return 0; |
| 248 | } |
| 249 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 250 | if (list_empty(&rdev->fence_drv.emited)) { |
| 251 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 252 | return 0; |
| 253 | } |
| 254 | fence = list_entry(rdev->fence_drv.emited.next, |
| 255 | struct radeon_fence, list); |
| 256 | radeon_fence_ref(fence); |
| 257 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 258 | r = radeon_fence_wait(fence, false); |
| 259 | radeon_fence_unref(&fence); |
| 260 | return r; |
| 261 | } |
| 262 | |
| 263 | int radeon_fence_wait_last(struct radeon_device *rdev) |
| 264 | { |
| 265 | unsigned long irq_flags; |
| 266 | struct radeon_fence *fence; |
| 267 | int r; |
| 268 | |
| 269 | if (rdev->gpu_lockup) { |
| 270 | return 0; |
| 271 | } |
| 272 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 273 | if (list_empty(&rdev->fence_drv.emited)) { |
| 274 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 275 | return 0; |
| 276 | } |
| 277 | fence = list_entry(rdev->fence_drv.emited.prev, |
| 278 | struct radeon_fence, list); |
| 279 | radeon_fence_ref(fence); |
| 280 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 281 | r = radeon_fence_wait(fence, false); |
| 282 | radeon_fence_unref(&fence); |
| 283 | return r; |
| 284 | } |
| 285 | |
| 286 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) |
| 287 | { |
| 288 | kref_get(&fence->kref); |
| 289 | return fence; |
| 290 | } |
| 291 | |
| 292 | void radeon_fence_unref(struct radeon_fence **fence) |
| 293 | { |
| 294 | struct radeon_fence *tmp = *fence; |
| 295 | |
| 296 | *fence = NULL; |
| 297 | if (tmp) { |
| 298 | kref_put(&tmp->kref, &radeon_fence_destroy); |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | void radeon_fence_process(struct radeon_device *rdev) |
| 303 | { |
| 304 | unsigned long irq_flags; |
| 305 | bool wake; |
| 306 | |
| 307 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 308 | wake = radeon_fence_poll_locked(rdev); |
| 309 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 310 | if (wake) { |
| 311 | wake_up_all(&rdev->fence_drv.queue); |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | int radeon_fence_driver_init(struct radeon_device *rdev) |
| 316 | { |
| 317 | unsigned long irq_flags; |
| 318 | int r; |
| 319 | |
| 320 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 321 | r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); |
| 322 | if (r) { |
| 323 | DRM_ERROR("Fence failed to get a scratch register."); |
| 324 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 325 | return r; |
| 326 | } |
| 327 | WREG32(rdev->fence_drv.scratch_reg, 0); |
| 328 | atomic_set(&rdev->fence_drv.seq, 0); |
| 329 | INIT_LIST_HEAD(&rdev->fence_drv.created); |
| 330 | INIT_LIST_HEAD(&rdev->fence_drv.emited); |
| 331 | INIT_LIST_HEAD(&rdev->fence_drv.signaled); |
| 332 | rdev->fence_drv.count_timeout = 0; |
| 333 | init_waitqueue_head(&rdev->fence_drv.queue); |
| 334 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 335 | if (radeon_debugfs_fence_init(rdev)) { |
| 336 | DRM_ERROR("Failed to register debugfs file for fence !\n"); |
| 337 | } |
| 338 | return 0; |
| 339 | } |
| 340 | |
| 341 | void radeon_fence_driver_fini(struct radeon_device *rdev) |
| 342 | { |
| 343 | unsigned long irq_flags; |
| 344 | |
| 345 | wake_up_all(&rdev->fence_drv.queue); |
| 346 | write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); |
| 347 | radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); |
| 348 | write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); |
| 349 | DRM_INFO("radeon: fence finalized\n"); |
| 350 | } |
| 351 | |
| 352 | |
| 353 | /* |
| 354 | * Fence debugfs |
| 355 | */ |
| 356 | #if defined(CONFIG_DEBUG_FS) |
| 357 | static int radeon_debugfs_fence_info(struct seq_file *m, void *data) |
| 358 | { |
| 359 | struct drm_info_node *node = (struct drm_info_node *)m->private; |
| 360 | struct drm_device *dev = node->minor->dev; |
| 361 | struct radeon_device *rdev = dev->dev_private; |
| 362 | struct radeon_fence *fence; |
| 363 | |
| 364 | seq_printf(m, "Last signaled fence 0x%08X\n", |
| 365 | RREG32(rdev->fence_drv.scratch_reg)); |
| 366 | if (!list_empty(&rdev->fence_drv.emited)) { |
| 367 | fence = list_entry(rdev->fence_drv.emited.prev, |
| 368 | struct radeon_fence, list); |
| 369 | seq_printf(m, "Last emited fence %p with 0x%08X\n", |
| 370 | fence, fence->seq); |
| 371 | } |
| 372 | return 0; |
| 373 | } |
| 374 | |
| 375 | static struct drm_info_list radeon_debugfs_fence_list[] = { |
| 376 | {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, |
| 377 | }; |
| 378 | #endif |
| 379 | |
| 380 | int radeon_debugfs_fence_init(struct radeon_device *rdev) |
| 381 | { |
| 382 | #if defined(CONFIG_DEBUG_FS) |
| 383 | return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); |
| 384 | #else |
| 385 | return 0; |
| 386 | #endif |
| 387 | } |