Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2007 Ben Skeggs. |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * Permission is hereby granted, free of charge, to any person obtaining |
| 6 | * a copy of this software and associated documentation files (the |
| 7 | * "Software"), to deal in the Software without restriction, including |
| 8 | * without limitation the rights to use, copy, modify, merge, publish, |
| 9 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 10 | * permit persons to whom the Software is furnished to do so, subject to |
| 11 | * the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice (including the |
| 14 | * next paragraph) shall be included in all copies or substantial |
| 15 | * portions of the Software. |
| 16 | * |
| 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
| 20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
| 21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 24 | * |
| 25 | */ |
| 26 | |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 27 | #include "nouveau_drm.h" |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 28 | #include "nouveau_dma.h" |
Ben Skeggs | 75c99da | 2010-01-08 10:57:39 +1000 | [diff] [blame] | 29 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 30 | void |
| 31 | OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords) |
| 32 | { |
| 33 | bool is_iomem; |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 34 | u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 35 | mem = &mem[chan->dma.cur]; |
| 36 | if (is_iomem) |
| 37 | memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4); |
| 38 | else |
| 39 | memcpy(mem, data, nr_dwords * 4); |
| 40 | chan->dma.cur += nr_dwords; |
| 41 | } |
| 42 | |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 43 | /* Fetch and adjust GPU GET pointer |
| 44 | * |
| 45 | * Returns: |
| 46 | * value >= 0, the adjusted GET pointer |
| 47 | * -EINVAL if GET pointer currently outside main push buffer |
| 48 | * -EBUSY if timeout exceeded |
| 49 | */ |
| 50 | static inline int |
Francisco Jerez | 4e03b4a | 2011-11-19 11:57:52 +0100 | [diff] [blame] | 51 | READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 52 | { |
Francisco Jerez | 4e03b4a | 2011-11-19 11:57:52 +0100 | [diff] [blame] | 53 | uint64_t val; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 54 | |
Ben Skeggs | a01ca78 | 2015-08-20 14:54:15 +1000 | [diff] [blame] | 55 | val = nvif_rd32(&chan->user, chan->user_get); |
Francisco Jerez | 4e03b4a | 2011-11-19 11:57:52 +0100 | [diff] [blame] | 56 | if (chan->user_get_hi) |
Ben Skeggs | a01ca78 | 2015-08-20 14:54:15 +1000 | [diff] [blame] | 57 | val |= (uint64_t)nvif_rd32(&chan->user, chan->user_get_hi) << 32; |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 58 | |
| 59 | /* reset counter as long as GET is still advancing, this is |
| 60 | * to avoid misdetecting a GPU lockup if the GPU happens to |
| 61 | * just be processing an operation that takes a long time |
| 62 | */ |
| 63 | if (val != *prev_get) { |
| 64 | *prev_get = val; |
| 65 | *timeout = 0; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 66 | } |
| 67 | |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 68 | if ((++*timeout & 0xff) == 0) { |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 69 | udelay(1); |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 70 | if (*timeout > 100000) |
| 71 | return -EBUSY; |
| 72 | } |
| 73 | |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 74 | if (val < chan->push.vma.offset || |
| 75 | val > chan->push.vma.offset + (chan->dma.max << 2)) |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 76 | return -EINVAL; |
| 77 | |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 78 | return (val - chan->push.vma.offset) >> 2; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 79 | } |
| 80 | |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 81 | void |
| 82 | nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo, |
Ben Skeggs | a1606a9 | 2010-02-12 10:27:35 +1000 | [diff] [blame] | 83 | int delta, int length) |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 84 | { |
Ben Skeggs | a01ca78 | 2015-08-20 14:54:15 +1000 | [diff] [blame] | 85 | struct nouveau_cli *cli = (void *)chan->user.client; |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 86 | struct nouveau_bo *pb = chan->push.buffer; |
Ben Skeggs | be83cd4 | 2015-01-14 15:36:34 +1000 | [diff] [blame] | 87 | struct nvkm_vma *vma; |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 88 | int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base; |
Ben Skeggs | 9f9f51f | 2011-06-07 13:23:47 +1000 | [diff] [blame] | 89 | u64 offset; |
| 90 | |
Ben Skeggs | 0ad7286 | 2014-08-10 04:10:22 +1000 | [diff] [blame] | 91 | vma = nouveau_bo_vma_find(bo, cli->vm); |
Ben Skeggs | 9f9f51f | 2011-06-07 13:23:47 +1000 | [diff] [blame] | 92 | BUG_ON(!vma); |
| 93 | offset = vma->offset + delta; |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 94 | |
| 95 | BUG_ON(chan->dma.ib_free < 1); |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 96 | |
Ben Skeggs | d87897d | 2010-02-12 11:11:54 +1000 | [diff] [blame] | 97 | nouveau_bo_wr32(pb, ip++, lower_32_bits(offset)); |
Ben Skeggs | a1606a9 | 2010-02-12 10:27:35 +1000 | [diff] [blame] | 98 | nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8); |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 99 | |
| 100 | chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max; |
Maarten Maathuis | ce48fa9 | 2010-02-25 20:00:38 +0100 | [diff] [blame] | 101 | |
Daniel Vetter | 85b2331 | 2013-12-11 11:34:45 +0100 | [diff] [blame] | 102 | mb(); |
Maarten Maathuis | ce48fa9 | 2010-02-25 20:00:38 +0100 | [diff] [blame] | 103 | /* Flush writes. */ |
| 104 | nouveau_bo_rd32(pb, 0); |
| 105 | |
Ben Skeggs | a01ca78 | 2015-08-20 14:54:15 +1000 | [diff] [blame] | 106 | nvif_wr32(&chan->user, 0x8c, chan->dma.ib_put); |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 107 | chan->dma.ib_free--; |
| 108 | } |
| 109 | |
| 110 | static int |
| 111 | nv50_dma_push_wait(struct nouveau_channel *chan, int count) |
| 112 | { |
| 113 | uint32_t cnt = 0, prev_get = 0; |
| 114 | |
| 115 | while (chan->dma.ib_free < count) { |
Ben Skeggs | a01ca78 | 2015-08-20 14:54:15 +1000 | [diff] [blame] | 116 | uint32_t get = nvif_rd32(&chan->user, 0x88); |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 117 | if (get != prev_get) { |
| 118 | prev_get = get; |
| 119 | cnt = 0; |
| 120 | } |
| 121 | |
| 122 | if ((++cnt & 0xff) == 0) { |
| 123 | DRM_UDELAY(1); |
| 124 | if (cnt > 100000) |
| 125 | return -EBUSY; |
| 126 | } |
| 127 | |
| 128 | chan->dma.ib_free = get - chan->dma.ib_put; |
| 129 | if (chan->dma.ib_free <= 0) |
Ben Skeggs | 62841ab | 2010-09-30 09:09:42 +1000 | [diff] [blame] | 130 | chan->dma.ib_free += chan->dma.ib_max; |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 131 | } |
| 132 | |
| 133 | return 0; |
| 134 | } |
| 135 | |
| 136 | static int |
| 137 | nv50_dma_wait(struct nouveau_channel *chan, int slots, int count) |
| 138 | { |
Francisco Jerez | 4e03b4a | 2011-11-19 11:57:52 +0100 | [diff] [blame] | 139 | uint64_t prev_get = 0; |
| 140 | int ret, cnt = 0; |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 141 | |
| 142 | ret = nv50_dma_push_wait(chan, slots + 1); |
| 143 | if (unlikely(ret)) |
| 144 | return ret; |
| 145 | |
| 146 | while (chan->dma.free < count) { |
| 147 | int get = READ_GET(chan, &prev_get, &cnt); |
| 148 | if (unlikely(get < 0)) { |
| 149 | if (get == -EINVAL) |
| 150 | continue; |
| 151 | |
| 152 | return get; |
| 153 | } |
| 154 | |
| 155 | if (get <= chan->dma.cur) { |
| 156 | chan->dma.free = chan->dma.max - chan->dma.cur; |
| 157 | if (chan->dma.free >= count) |
| 158 | break; |
| 159 | |
| 160 | FIRE_RING(chan); |
| 161 | do { |
| 162 | get = READ_GET(chan, &prev_get, &cnt); |
| 163 | if (unlikely(get < 0)) { |
| 164 | if (get == -EINVAL) |
| 165 | continue; |
| 166 | return get; |
| 167 | } |
| 168 | } while (get == 0); |
| 169 | chan->dma.cur = 0; |
| 170 | chan->dma.put = 0; |
| 171 | } |
| 172 | |
| 173 | chan->dma.free = get - chan->dma.cur - 1; |
| 174 | } |
| 175 | |
| 176 | return 0; |
| 177 | } |
| 178 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 179 | int |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 180 | nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 181 | { |
Francisco Jerez | 4e03b4a | 2011-11-19 11:57:52 +0100 | [diff] [blame] | 182 | uint64_t prev_get = 0; |
| 183 | int cnt = 0, get; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 184 | |
Ben Skeggs | 9a391ad | 2010-02-11 16:37:26 +1000 | [diff] [blame] | 185 | if (chan->dma.ib_max) |
| 186 | return nv50_dma_wait(chan, slots, size); |
| 187 | |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 188 | while (chan->dma.free < size) { |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 189 | get = READ_GET(chan, &prev_get, &cnt); |
| 190 | if (unlikely(get == -EBUSY)) |
| 191 | return -EBUSY; |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 192 | |
| 193 | /* loop until we have a usable GET pointer. the value |
| 194 | * we read from the GPU may be outside the main ring if |
| 195 | * PFIFO is processing a buffer called from the main ring, |
| 196 | * discard these values until something sensible is seen. |
| 197 | * |
| 198 | * the other case we discard GET is while the GPU is fetching |
| 199 | * from the SKIPS area, so the code below doesn't have to deal |
| 200 | * with some fun corner cases. |
| 201 | */ |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 202 | if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS) |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 203 | continue; |
| 204 | |
| 205 | if (get <= chan->dma.cur) { |
| 206 | /* engine is fetching behind us, or is completely |
| 207 | * idle (GET == PUT) so we have free space up until |
| 208 | * the end of the push buffer |
| 209 | * |
| 210 | * we can only hit that path once per call due to |
| 211 | * looping back to the beginning of the push buffer, |
| 212 | * we'll hit the fetching-ahead-of-us path from that |
| 213 | * point on. |
| 214 | * |
| 215 | * the *one* exception to that rule is if we read |
| 216 | * GET==PUT, in which case the below conditional will |
| 217 | * always succeed and break us out of the wait loop. |
| 218 | */ |
| 219 | chan->dma.free = chan->dma.max - chan->dma.cur; |
| 220 | if (chan->dma.free >= size) |
| 221 | break; |
| 222 | |
| 223 | /* not enough space left at the end of the push buffer, |
| 224 | * instruct the GPU to jump back to the start right |
| 225 | * after processing the currently pending commands. |
| 226 | */ |
Ben Skeggs | ebb945a | 2012-07-20 08:17:34 +1000 | [diff] [blame] | 227 | OUT_RING(chan, chan->push.vma.offset | 0x20000000); |
Ben Skeggs | ba59953 | 2010-01-15 12:08:57 +1000 | [diff] [blame] | 228 | |
| 229 | /* wait for GET to depart from the skips area. |
| 230 | * prevents writing GET==PUT and causing a race |
| 231 | * condition that causes us to think the GPU is |
| 232 | * idle when it's not. |
| 233 | */ |
| 234 | do { |
| 235 | get = READ_GET(chan, &prev_get, &cnt); |
| 236 | if (unlikely(get == -EBUSY)) |
| 237 | return -EBUSY; |
| 238 | if (unlikely(get == -EINVAL)) |
| 239 | continue; |
| 240 | } while (get <= NOUVEAU_DMA_SKIPS); |
Ben Skeggs | 6ee7386 | 2009-12-11 19:24:15 +1000 | [diff] [blame] | 241 | WRITE_PUT(NOUVEAU_DMA_SKIPS); |
| 242 | |
| 243 | /* we're now submitting commands at the start of |
| 244 | * the push buffer. |
| 245 | */ |
| 246 | chan->dma.cur = |
| 247 | chan->dma.put = NOUVEAU_DMA_SKIPS; |
| 248 | } |
| 249 | |
| 250 | /* engine fetching ahead of us, we have space up until the |
| 251 | * current GET pointer. the "- 1" is to ensure there's |
| 252 | * space left to emit a jump back to the beginning of the |
| 253 | * push buffer if we require it. we can never get GET == PUT |
| 254 | * here, so this is safe. |
| 255 | */ |
| 256 | chan->dma.free = get - chan->dma.cur - 1; |
| 257 | } |
| 258 | |
| 259 | return 0; |
| 260 | } |
| 261 | |