Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2007 Ben Skeggs. |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining |
| 6 | * a copy of this software and associated documentation files (the |
| 7 | * "Software"), to deal in the Software without restriction, including |
| 8 | * without limitation the rights to use, copy, modify, merge, publish, |
| 9 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 10 | * permit persons to whom the Software is furnished to do so, subject to |
| 11 | * the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice (including the |
| 14 | * next paragraph) shall be included in all copies or substantial |
| 15 | * portions of the Software. |
| 16 | * |
| 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
| 20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
| 21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 24 | * |
| 25 | */ |
| 26 | |
| 27 | #include "drmP.h" |
| 28 | #include "drm.h" |
| 29 | |
Marcin Slusarz | bd35fe5 | 2011-03-09 14:22:19 +0100 | [diff] [blame^] | 30 | #include <linux/ktime.h> |
| 31 | #include <linux/hrtimer.h> |
| 32 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 33 | #include "nouveau_drv.h" |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 34 | #include "nouveau_ramht.h" |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 35 | #include "nouveau_dma.h" |
| 36 | |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 37 | #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10) |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 38 | #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 39 | |
| 40 | struct nouveau_fence { |
| 41 | struct nouveau_channel *channel; |
| 42 | struct kref refcount; |
| 43 | struct list_head entry; |
| 44 | |
| 45 | uint32_t sequence; |
| 46 | bool signalled; |
Francisco Jerez | 8ac3891 | 2010-09-21 20:49:39 +0200 | [diff] [blame] | 47 | |
| 48 | void (*work)(void *priv, bool signalled); |
| 49 | void *priv; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 50 | }; |
| 51 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 52 | struct nouveau_semaphore { |
| 53 | struct kref ref; |
| 54 | struct drm_device *dev; |
| 55 | struct drm_mm_node *mem; |
| 56 | }; |
| 57 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 58 | static inline struct nouveau_fence * |
| 59 | nouveau_fence(void *sync_obj) |
| 60 | { |
| 61 | return (struct nouveau_fence *)sync_obj; |
| 62 | } |
| 63 | |
| 64 | static void |
| 65 | nouveau_fence_del(struct kref *ref) |
| 66 | { |
| 67 | struct nouveau_fence *fence = |
| 68 | container_of(ref, struct nouveau_fence, refcount); |
| 69 | |
Francisco Jerez | 2a6789a | 2010-10-18 03:56:14 +0200 | [diff] [blame] | 70 | nouveau_channel_ref(NULL, &fence->channel); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 71 | kfree(fence); |
| 72 | } |
| 73 | |
| 74 | void |
| 75 | nouveau_fence_update(struct nouveau_channel *chan) |
| 76 | { |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 77 | struct drm_device *dev = chan->dev; |
| 78 | struct nouveau_fence *tmp, *fence; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 79 | uint32_t sequence; |
| 80 | |
Francisco Jerez | 3ba6462 | 2010-08-28 17:56:33 +0200 | [diff] [blame] | 81 | spin_lock(&chan->fence.lock); |
| 82 | |
Francisco Jerez | 937c347 | 2010-12-08 02:35:45 +0100 | [diff] [blame] | 83 | /* Fetch the last sequence if the channel is still up and running */ |
| 84 | if (likely(!list_empty(&chan->fence.pending))) { |
| 85 | if (USE_REFCNT(dev)) |
| 86 | sequence = nvchan_rd32(chan, 0x48); |
| 87 | else |
| 88 | sequence = atomic_read(&chan->fence.last_sequence_irq); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 89 | |
Francisco Jerez | 937c347 | 2010-12-08 02:35:45 +0100 | [diff] [blame] | 90 | if (chan->fence.sequence_ack == sequence) |
| 91 | goto out; |
| 92 | chan->fence.sequence_ack = sequence; |
| 93 | } |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 94 | |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 95 | list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 96 | sequence = fence->sequence; |
| 97 | fence->signalled = true; |
| 98 | list_del(&fence->entry); |
Francisco Jerez | 8ac3891 | 2010-09-21 20:49:39 +0200 | [diff] [blame] | 99 | |
| 100 | if (unlikely(fence->work)) |
| 101 | fence->work(fence->priv, true); |
| 102 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 103 | kref_put(&fence->refcount, nouveau_fence_del); |
| 104 | |
| 105 | if (sequence == chan->fence.sequence_ack) |
| 106 | break; |
| 107 | } |
Francisco Jerez | 3ba6462 | 2010-08-28 17:56:33 +0200 | [diff] [blame] | 108 | out: |
Ben Skeggs | 047d1d3 | 2010-05-31 12:00:43 +1000 | [diff] [blame] | 109 | spin_unlock(&chan->fence.lock); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | int |
| 113 | nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence, |
| 114 | bool emit) |
| 115 | { |
| 116 | struct nouveau_fence *fence; |
| 117 | int ret = 0; |
| 118 | |
| 119 | fence = kzalloc(sizeof(*fence), GFP_KERNEL); |
| 120 | if (!fence) |
| 121 | return -ENOMEM; |
| 122 | kref_init(&fence->refcount); |
Francisco Jerez | 2a6789a | 2010-10-18 03:56:14 +0200 | [diff] [blame] | 123 | nouveau_channel_ref(chan, &fence->channel); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 124 | |
| 125 | if (emit) |
| 126 | ret = nouveau_fence_emit(fence); |
| 127 | |
| 128 | if (ret) |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 129 | nouveau_fence_unref(&fence); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 130 | *pfence = fence; |
| 131 | return ret; |
| 132 | } |
| 133 | |
| 134 | struct nouveau_channel * |
| 135 | nouveau_fence_channel(struct nouveau_fence *fence) |
| 136 | { |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 137 | return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 138 | } |
| 139 | |
| 140 | int |
| 141 | nouveau_fence_emit(struct nouveau_fence *fence) |
| 142 | { |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 143 | struct nouveau_channel *chan = fence->channel; |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 144 | struct drm_device *dev = chan->dev; |
Ben Skeggs | 529c495 | 2010-11-24 10:30:22 +1000 | [diff] [blame] | 145 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 146 | int ret; |
| 147 | |
| 148 | ret = RING_SPACE(chan, 2); |
| 149 | if (ret) |
| 150 | return ret; |
| 151 | |
| 152 | if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) { |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 153 | nouveau_fence_update(chan); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 154 | |
| 155 | BUG_ON(chan->fence.sequence == |
| 156 | chan->fence.sequence_ack - 1); |
| 157 | } |
| 158 | |
| 159 | fence->sequence = ++chan->fence.sequence; |
| 160 | |
| 161 | kref_get(&fence->refcount); |
Ben Skeggs | 047d1d3 | 2010-05-31 12:00:43 +1000 | [diff] [blame] | 162 | spin_lock(&chan->fence.lock); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 163 | list_add_tail(&fence->entry, &chan->fence.pending); |
Ben Skeggs | 047d1d3 | 2010-05-31 12:00:43 +1000 | [diff] [blame] | 164 | spin_unlock(&chan->fence.lock); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 165 | |
Ben Skeggs | 529c495 | 2010-11-24 10:30:22 +1000 | [diff] [blame] | 166 | if (USE_REFCNT(dev)) { |
| 167 | if (dev_priv->card_type < NV_C0) |
| 168 | BEGIN_RING(chan, NvSubSw, 0x0050, 1); |
| 169 | else |
Ben Skeggs | 2a55c9a | 2010-12-30 11:53:48 +1000 | [diff] [blame] | 170 | BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1); |
Ben Skeggs | 529c495 | 2010-11-24 10:30:22 +1000 | [diff] [blame] | 171 | } else { |
| 172 | BEGIN_RING(chan, NvSubSw, 0x0150, 1); |
| 173 | } |
| 174 | OUT_RING (chan, fence->sequence); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 175 | FIRE_RING(chan); |
| 176 | |
| 177 | return 0; |
| 178 | } |
| 179 | |
| 180 | void |
Francisco Jerez | 8ac3891 | 2010-09-21 20:49:39 +0200 | [diff] [blame] | 181 | nouveau_fence_work(struct nouveau_fence *fence, |
| 182 | void (*work)(void *priv, bool signalled), |
| 183 | void *priv) |
| 184 | { |
| 185 | BUG_ON(fence->work); |
| 186 | |
| 187 | spin_lock(&fence->channel->fence.lock); |
| 188 | |
| 189 | if (fence->signalled) { |
| 190 | work(priv, true); |
| 191 | } else { |
| 192 | fence->work = work; |
| 193 | fence->priv = priv; |
| 194 | } |
| 195 | |
| 196 | spin_unlock(&fence->channel->fence.lock); |
| 197 | } |
| 198 | |
| 199 | void |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 200 | __nouveau_fence_unref(void **sync_obj) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 201 | { |
| 202 | struct nouveau_fence *fence = nouveau_fence(*sync_obj); |
| 203 | |
| 204 | if (fence) |
| 205 | kref_put(&fence->refcount, nouveau_fence_del); |
| 206 | *sync_obj = NULL; |
| 207 | } |
| 208 | |
| 209 | void * |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 210 | __nouveau_fence_ref(void *sync_obj) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 211 | { |
| 212 | struct nouveau_fence *fence = nouveau_fence(sync_obj); |
| 213 | |
| 214 | kref_get(&fence->refcount); |
| 215 | return sync_obj; |
| 216 | } |
| 217 | |
| 218 | bool |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 219 | __nouveau_fence_signalled(void *sync_obj, void *sync_arg) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 220 | { |
| 221 | struct nouveau_fence *fence = nouveau_fence(sync_obj); |
| 222 | struct nouveau_channel *chan = fence->channel; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 223 | |
| 224 | if (fence->signalled) |
| 225 | return true; |
| 226 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 227 | nouveau_fence_update(chan); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 228 | return fence->signalled; |
| 229 | } |
| 230 | |
| 231 | int |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 232 | __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 233 | { |
| 234 | unsigned long timeout = jiffies + (3 * DRM_HZ); |
Marcin Slusarz | bd35fe5 | 2011-03-09 14:22:19 +0100 | [diff] [blame^] | 235 | unsigned long sleep_time = NSEC_PER_MSEC / 1000; |
| 236 | ktime_t t; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 237 | int ret = 0; |
| 238 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 239 | while (1) { |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 240 | if (__nouveau_fence_signalled(sync_obj, sync_arg)) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 241 | break; |
| 242 | |
| 243 | if (time_after_eq(jiffies, timeout)) { |
| 244 | ret = -EBUSY; |
| 245 | break; |
| 246 | } |
| 247 | |
Kulikov Vasiliy | 0599111 | 2010-07-26 12:23:54 +0400 | [diff] [blame] | 248 | __set_current_state(intr ? TASK_INTERRUPTIBLE |
| 249 | : TASK_UNINTERRUPTIBLE); |
Marcin Slusarz | bd35fe5 | 2011-03-09 14:22:19 +0100 | [diff] [blame^] | 250 | if (lazy) { |
| 251 | t = ktime_set(0, sleep_time); |
| 252 | schedule_hrtimeout(&t, HRTIMER_MODE_REL); |
| 253 | sleep_time *= 2; |
| 254 | if (sleep_time > NSEC_PER_MSEC) |
| 255 | sleep_time = NSEC_PER_MSEC; |
| 256 | } |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 257 | |
| 258 | if (intr && signal_pending(current)) { |
Ben Skeggs | 9ddc8c5 | 2009-12-15 11:04:25 +1000 | [diff] [blame] | 259 | ret = -ERESTARTSYS; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 260 | break; |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | __set_current_state(TASK_RUNNING); |
| 265 | |
| 266 | return ret; |
| 267 | } |
| 268 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 269 | static struct nouveau_semaphore * |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 270 | semaphore_alloc(struct drm_device *dev) |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 271 | { |
| 272 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
| 273 | struct nouveau_semaphore *sema; |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 274 | int size = (dev_priv->chipset < 0x84) ? 4 : 16; |
| 275 | int ret, i; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 276 | |
| 277 | if (!USE_SEMA(dev)) |
| 278 | return NULL; |
| 279 | |
| 280 | sema = kmalloc(sizeof(*sema), GFP_KERNEL); |
| 281 | if (!sema) |
| 282 | goto fail; |
| 283 | |
Francisco Jerez | 907af60 | 2010-10-09 04:02:09 +0200 | [diff] [blame] | 284 | ret = drm_mm_pre_get(&dev_priv->fence.heap); |
| 285 | if (ret) |
| 286 | goto fail; |
| 287 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 288 | spin_lock(&dev_priv->fence.lock); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 289 | sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 290 | if (sema->mem) |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 291 | sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 292 | spin_unlock(&dev_priv->fence.lock); |
| 293 | |
| 294 | if (!sema->mem) |
| 295 | goto fail; |
| 296 | |
| 297 | kref_init(&sema->ref); |
| 298 | sema->dev = dev; |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 299 | for (i = sema->mem->start; i < sema->mem->start + size; i += 4) |
| 300 | nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 301 | |
| 302 | return sema; |
| 303 | fail: |
| 304 | kfree(sema); |
| 305 | return NULL; |
| 306 | } |
| 307 | |
| 308 | static void |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 309 | semaphore_free(struct kref *ref) |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 310 | { |
| 311 | struct nouveau_semaphore *sema = |
| 312 | container_of(ref, struct nouveau_semaphore, ref); |
| 313 | struct drm_nouveau_private *dev_priv = sema->dev->dev_private; |
| 314 | |
| 315 | spin_lock(&dev_priv->fence.lock); |
| 316 | drm_mm_put_block(sema->mem); |
| 317 | spin_unlock(&dev_priv->fence.lock); |
| 318 | |
| 319 | kfree(sema); |
| 320 | } |
| 321 | |
| 322 | static void |
| 323 | semaphore_work(void *priv, bool signalled) |
| 324 | { |
| 325 | struct nouveau_semaphore *sema = priv; |
| 326 | struct drm_nouveau_private *dev_priv = sema->dev->dev_private; |
| 327 | |
| 328 | if (unlikely(!signalled)) |
| 329 | nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1); |
| 330 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 331 | kref_put(&sema->ref, semaphore_free); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 332 | } |
| 333 | |
| 334 | static int |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 335 | semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema) |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 336 | { |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 337 | struct drm_nouveau_private *dev_priv = chan->dev->dev_private; |
| 338 | struct nouveau_fence *fence = NULL; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 339 | int ret; |
| 340 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 341 | if (dev_priv->chipset < 0x84) { |
Ben Skeggs | ec23802 | 2011-02-02 14:57:05 +1000 | [diff] [blame] | 342 | ret = RING_SPACE(chan, 3); |
| 343 | if (ret) |
| 344 | return ret; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 345 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 346 | BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 2); |
| 347 | OUT_RING (chan, sema->mem->start); |
| 348 | OUT_RING (chan, 1); |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 349 | } else |
| 350 | if (dev_priv->chipset < 0xc0) { |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 351 | struct nouveau_vma *vma = &dev_priv->fence.bo->vma; |
| 352 | u64 offset = vma->offset + sema->mem->start; |
| 353 | |
Ben Skeggs | ec23802 | 2011-02-02 14:57:05 +1000 | [diff] [blame] | 354 | ret = RING_SPACE(chan, 5); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 355 | if (ret) |
| 356 | return ret; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 357 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 358 | BEGIN_RING(chan, NvSubSw, 0x0010, 4); |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 359 | OUT_RING (chan, upper_32_bits(offset)); |
| 360 | OUT_RING (chan, lower_32_bits(offset)); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 361 | OUT_RING (chan, 1); |
| 362 | OUT_RING (chan, 1); /* ACQUIRE_EQ */ |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 363 | } else { |
| 364 | struct nouveau_vma *vma = &dev_priv->fence.bo->vma; |
| 365 | u64 offset = vma->offset + sema->mem->start; |
| 366 | |
| 367 | ret = RING_SPACE(chan, 5); |
| 368 | if (ret) |
| 369 | return ret; |
| 370 | |
| 371 | BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); |
| 372 | OUT_RING (chan, upper_32_bits(offset)); |
| 373 | OUT_RING (chan, lower_32_bits(offset)); |
| 374 | OUT_RING (chan, 1); |
| 375 | OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */ |
Francisco Jerez | 8af29cc | 2010-10-02 17:04:46 +0200 | [diff] [blame] | 376 | } |
| 377 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 378 | /* Delay semaphore destruction until its work is done */ |
| 379 | ret = nouveau_fence_new(chan, &fence, true); |
| 380 | if (ret) |
| 381 | return ret; |
| 382 | |
| 383 | kref_get(&sema->ref); |
| 384 | nouveau_fence_work(fence, semaphore_work, sema); |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 385 | nouveau_fence_unref(&fence); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 386 | return 0; |
| 387 | } |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 388 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 389 | static int |
| 390 | semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema) |
| 391 | { |
| 392 | struct drm_nouveau_private *dev_priv = chan->dev->dev_private; |
| 393 | struct nouveau_fence *fence = NULL; |
| 394 | int ret; |
| 395 | |
| 396 | if (dev_priv->chipset < 0x84) { |
Ben Skeggs | ec23802 | 2011-02-02 14:57:05 +1000 | [diff] [blame] | 397 | ret = RING_SPACE(chan, 4); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 398 | if (ret) |
| 399 | return ret; |
| 400 | |
| 401 | BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1); |
| 402 | OUT_RING (chan, sema->mem->start); |
| 403 | BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1); |
| 404 | OUT_RING (chan, 1); |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 405 | } else |
| 406 | if (dev_priv->chipset < 0xc0) { |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 407 | struct nouveau_vma *vma = &dev_priv->fence.bo->vma; |
| 408 | u64 offset = vma->offset + sema->mem->start; |
| 409 | |
Ben Skeggs | ec23802 | 2011-02-02 14:57:05 +1000 | [diff] [blame] | 410 | ret = RING_SPACE(chan, 5); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 411 | if (ret) |
| 412 | return ret; |
| 413 | |
| 414 | BEGIN_RING(chan, NvSubSw, 0x0010, 4); |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 415 | OUT_RING (chan, upper_32_bits(offset)); |
| 416 | OUT_RING (chan, lower_32_bits(offset)); |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 417 | OUT_RING (chan, 1); |
| 418 | OUT_RING (chan, 2); /* RELEASE */ |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 419 | } else { |
| 420 | struct nouveau_vma *vma = &dev_priv->fence.bo->vma; |
| 421 | u64 offset = vma->offset + sema->mem->start; |
| 422 | |
| 423 | ret = RING_SPACE(chan, 5); |
| 424 | if (ret) |
| 425 | return ret; |
| 426 | |
| 427 | BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); |
| 428 | OUT_RING (chan, upper_32_bits(offset)); |
| 429 | OUT_RING (chan, lower_32_bits(offset)); |
| 430 | OUT_RING (chan, 1); |
| 431 | OUT_RING (chan, 0x1002); /* RELEASE */ |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 432 | } |
| 433 | |
| 434 | /* Delay semaphore destruction until its work is done */ |
| 435 | ret = nouveau_fence_new(chan, &fence, true); |
| 436 | if (ret) |
| 437 | return ret; |
| 438 | |
| 439 | kref_get(&sema->ref); |
| 440 | nouveau_fence_work(fence, semaphore_work, sema); |
| 441 | nouveau_fence_unref(&fence); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 442 | return 0; |
| 443 | } |
| 444 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 445 | int |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 446 | nouveau_fence_sync(struct nouveau_fence *fence, |
| 447 | struct nouveau_channel *wchan) |
| 448 | { |
| 449 | struct nouveau_channel *chan = nouveau_fence_channel(fence); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 450 | struct drm_device *dev = wchan->dev; |
| 451 | struct nouveau_semaphore *sema; |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 452 | int ret = 0; |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 453 | |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 454 | if (likely(!chan || chan == wchan || |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 455 | nouveau_fence_signalled(fence))) |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 456 | goto out; |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 457 | |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 458 | sema = semaphore_alloc(dev); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 459 | if (!sema) { |
| 460 | /* Early card or broken userspace, fall back to |
| 461 | * software sync. */ |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 462 | ret = nouveau_fence_wait(fence, true, false); |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 463 | goto out; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 464 | } |
| 465 | |
Ben Skeggs | 08cd3d4 | 2010-10-12 08:01:59 +1000 | [diff] [blame] | 466 | /* try to take chan's mutex, if we can't take it right away |
Ben Skeggs | cff5c13 | 2010-10-06 16:16:59 +1000 | [diff] [blame] | 467 | * we have to fallback to software sync to prevent locking |
| 468 | * order issues |
| 469 | */ |
Ben Skeggs | 08cd3d4 | 2010-10-12 08:01:59 +1000 | [diff] [blame] | 470 | if (!mutex_trylock(&chan->mutex)) { |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 471 | ret = nouveau_fence_wait(fence, true, false); |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 472 | goto out_unref; |
Ben Skeggs | cff5c13 | 2010-10-06 16:16:59 +1000 | [diff] [blame] | 473 | } |
| 474 | |
Francisco Jerez | 8af29cc | 2010-10-02 17:04:46 +0200 | [diff] [blame] | 475 | /* Make wchan wait until it gets signalled */ |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 476 | ret = semaphore_acquire(wchan, sema); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 477 | if (ret) |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 478 | goto out_unlock; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 479 | |
Francisco Jerez | 8af29cc | 2010-10-02 17:04:46 +0200 | [diff] [blame] | 480 | /* Signal the semaphore from chan */ |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 481 | ret = semaphore_release(chan, sema); |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 482 | |
| 483 | out_unlock: |
Ben Skeggs | 08cd3d4 | 2010-10-12 08:01:59 +1000 | [diff] [blame] | 484 | mutex_unlock(&chan->mutex); |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 485 | out_unref: |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 486 | kref_put(&sema->ref, semaphore_free); |
Francisco Jerez | 2b478add | 2010-10-18 03:56:40 +0200 | [diff] [blame] | 487 | out: |
| 488 | if (chan) |
| 489 | nouveau_channel_put_unlocked(&chan); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 490 | return ret; |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 491 | } |
| 492 | |
| 493 | int |
Marcin Slusarz | 382d62e | 2010-10-20 21:50:24 +0200 | [diff] [blame] | 494 | __nouveau_fence_flush(void *sync_obj, void *sync_arg) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 495 | { |
| 496 | return 0; |
| 497 | } |
| 498 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 499 | int |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 500 | nouveau_fence_channel_init(struct nouveau_channel *chan) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 501 | { |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 502 | struct drm_device *dev = chan->dev; |
| 503 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 504 | struct nouveau_gpuobj *obj = NULL; |
| 505 | int ret; |
| 506 | |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 507 | if (dev_priv->card_type >= NV_C0) |
| 508 | goto out_initialised; |
| 509 | |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 510 | /* Create an NV_SW object for various sync purposes */ |
Ben Skeggs | ceac309 | 2010-11-23 10:10:24 +1000 | [diff] [blame] | 511 | ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW); |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 512 | if (ret) |
| 513 | return ret; |
| 514 | |
Ben Skeggs | 529c495 | 2010-11-24 10:30:22 +1000 | [diff] [blame] | 515 | /* we leave subchannel empty for nvc0 */ |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 516 | ret = RING_SPACE(chan, 2); |
| 517 | if (ret) |
| 518 | return ret; |
| 519 | BEGIN_RING(chan, NvSubSw, 0, 1); |
| 520 | OUT_RING(chan, NvSw); |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 521 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 522 | /* Create a DMA object for the shared cross-channel sync area. */ |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 523 | if (USE_SEMA(dev) && dev_priv->chipset < 0x84) { |
Ben Skeggs | a8b214f | 2010-12-03 09:05:20 +1000 | [diff] [blame] | 524 | struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 525 | |
| 526 | ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, |
| 527 | mem->start << PAGE_SHIFT, |
Ben Skeggs | a8b214f | 2010-12-03 09:05:20 +1000 | [diff] [blame] | 528 | mem->size, NV_MEM_ACCESS_RW, |
Ben Skeggs | 7f4a195 | 2010-11-16 11:50:09 +1000 | [diff] [blame] | 529 | NV_MEM_TARGET_VRAM, &obj); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 530 | if (ret) |
| 531 | return ret; |
| 532 | |
| 533 | ret = nouveau_ramht_insert(chan, NvSema, obj); |
| 534 | nouveau_gpuobj_ref(NULL, &obj); |
| 535 | if (ret) |
| 536 | return ret; |
| 537 | |
| 538 | ret = RING_SPACE(chan, 2); |
| 539 | if (ret) |
| 540 | return ret; |
| 541 | BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); |
| 542 | OUT_RING(chan, NvSema); |
Ben Skeggs | e3b7ed5 | 2011-02-02 13:21:57 +1000 | [diff] [blame] | 543 | } else { |
| 544 | ret = RING_SPACE(chan, 2); |
| 545 | if (ret) |
| 546 | return ret; |
| 547 | BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); |
| 548 | OUT_RING (chan, chan->vram_handle); /* whole VM */ |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 549 | } |
| 550 | |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 551 | FIRE_RING(chan); |
| 552 | |
Ben Skeggs | cb1d771 | 2011-01-28 13:44:32 +1000 | [diff] [blame] | 553 | out_initialised: |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 554 | INIT_LIST_HEAD(&chan->fence.pending); |
| 555 | spin_lock_init(&chan->fence.lock); |
Ben Skeggs | 047d1d3 | 2010-05-31 12:00:43 +1000 | [diff] [blame] | 556 | atomic_set(&chan->fence.last_sequence_irq, 0); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 557 | return 0; |
| 558 | } |
| 559 | |
| 560 | void |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 561 | nouveau_fence_channel_fini(struct nouveau_channel *chan) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 562 | { |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 563 | struct nouveau_fence *tmp, *fence; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 564 | |
Francisco Jerez | 889fa93 | 2010-10-18 03:57:19 +0200 | [diff] [blame] | 565 | spin_lock(&chan->fence.lock); |
| 566 | |
Francisco Jerez | 2730723 | 2010-09-21 18:57:11 +0200 | [diff] [blame] | 567 | list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 568 | fence->signalled = true; |
| 569 | list_del(&fence->entry); |
Francisco Jerez | 8ac3891 | 2010-09-21 20:49:39 +0200 | [diff] [blame] | 570 | |
| 571 | if (unlikely(fence->work)) |
| 572 | fence->work(fence->priv, false); |
| 573 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 574 | kref_put(&fence->refcount, nouveau_fence_del); |
| 575 | } |
Francisco Jerez | 889fa93 | 2010-10-18 03:57:19 +0200 | [diff] [blame] | 576 | |
| 577 | spin_unlock(&chan->fence.lock); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 578 | } |
| 579 | |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 580 | int |
| 581 | nouveau_fence_init(struct drm_device *dev) |
| 582 | { |
| 583 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 584 | int size = (dev_priv->chipset < 0x84) ? 4096 : 16384; |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 585 | int ret; |
| 586 | |
| 587 | /* Create a shared VRAM heap for cross-channel sync. */ |
| 588 | if (USE_SEMA(dev)) { |
Ben Skeggs | c3b90a7 | 2011-01-28 12:08:29 +1000 | [diff] [blame] | 589 | ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM, |
Ben Skeggs | d550c41 | 2011-02-16 08:41:56 +1000 | [diff] [blame] | 590 | 0, 0, &dev_priv->fence.bo); |
Francisco Jerez | 0c6c1c2 | 2010-09-22 00:58:54 +0200 | [diff] [blame] | 591 | if (ret) |
| 592 | return ret; |
| 593 | |
| 594 | ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM); |
| 595 | if (ret) |
| 596 | goto fail; |
| 597 | |
| 598 | ret = nouveau_bo_map(dev_priv->fence.bo); |
| 599 | if (ret) |
| 600 | goto fail; |
| 601 | |
| 602 | ret = drm_mm_init(&dev_priv->fence.heap, 0, |
| 603 | dev_priv->fence.bo->bo.mem.size); |
| 604 | if (ret) |
| 605 | goto fail; |
| 606 | |
| 607 | spin_lock_init(&dev_priv->fence.lock); |
| 608 | } |
| 609 | |
| 610 | return 0; |
| 611 | fail: |
| 612 | nouveau_bo_unmap(dev_priv->fence.bo); |
| 613 | nouveau_bo_ref(NULL, &dev_priv->fence.bo); |
| 614 | return ret; |
| 615 | } |
| 616 | |
| 617 | void |
| 618 | nouveau_fence_fini(struct drm_device *dev) |
| 619 | { |
| 620 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
| 621 | |
| 622 | if (USE_SEMA(dev)) { |
| 623 | drm_mm_takedown(&dev_priv->fence.heap); |
| 624 | nouveau_bo_unmap(dev_priv->fence.bo); |
| 625 | nouveau_bo_unpin(dev_priv->fence.bo); |
| 626 | nouveau_bo_ref(NULL, &dev_priv->fence.bo); |
| 627 | } |
| 628 | } |