blob: 808ea4d5b962f8c57803fbbf104d215abe547a21 [file] [log] [blame]
Chris Wilsone68a1392016-09-09 14:11:41 +01001/*
2 * (C) Copyright 2016 Intel Corporation
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; version 2
7 * of the License.
8 */
9
10#include <linux/slab.h>
Chris Wilsonf54d1862016-10-25 13:00:45 +010011#include <linux/dma-fence.h>
Chris Wilson81c0ed22017-09-11 09:41:26 +010012#include <linux/irq_work.h>
Chris Wilsone68a1392016-09-09 14:11:41 +010013#include <linux/reservation.h>
14
15#include "i915_sw_fence.h"
Chris Wilson47624cc2017-05-17 13:09:57 +010016#include "i915_selftest.h"
Chris Wilsone68a1392016-09-09 14:11:41 +010017
Chris Wilson7e941862016-10-28 13:58:25 +010018#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
19
Chris Wilsone68a1392016-09-09 14:11:41 +010020static DEFINE_SPINLOCK(i915_sw_fence_lock);
21
Chris Wilsonfc158402016-11-25 13:17:18 +000022enum {
23 DEBUG_FENCE_IDLE = 0,
24 DEBUG_FENCE_NOTIFY,
25};
26
27#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
28
29static void *i915_sw_fence_debug_hint(void *addr)
30{
31 return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
32}
33
34static struct debug_obj_descr i915_sw_fence_debug_descr = {
35 .name = "i915_sw_fence",
36 .debug_hint = i915_sw_fence_debug_hint,
37};
38
39static inline void debug_fence_init(struct i915_sw_fence *fence)
40{
41 debug_object_init(fence, &i915_sw_fence_debug_descr);
42}
43
44static inline void debug_fence_activate(struct i915_sw_fence *fence)
45{
46 debug_object_activate(fence, &i915_sw_fence_debug_descr);
47}
48
49static inline void debug_fence_set_state(struct i915_sw_fence *fence,
50 int old, int new)
51{
52 debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
53}
54
55static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
56{
57 debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
58}
59
60static inline void debug_fence_destroy(struct i915_sw_fence *fence)
61{
62 debug_object_destroy(fence, &i915_sw_fence_debug_descr);
63}
64
65static inline void debug_fence_free(struct i915_sw_fence *fence)
66{
67 debug_object_free(fence, &i915_sw_fence_debug_descr);
Chris Wilson6f13f292017-01-13 21:43:35 +000068 smp_wmb(); /* flush the change in state before reallocation */
Chris Wilsonfc158402016-11-25 13:17:18 +000069}
70
71static inline void debug_fence_assert(struct i915_sw_fence *fence)
72{
73 debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
74}
75
76#else
77
78static inline void debug_fence_init(struct i915_sw_fence *fence)
79{
80}
81
82static inline void debug_fence_activate(struct i915_sw_fence *fence)
83{
84}
85
86static inline void debug_fence_set_state(struct i915_sw_fence *fence,
87 int old, int new)
88{
89}
90
91static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
92{
93}
94
95static inline void debug_fence_destroy(struct i915_sw_fence *fence)
96{
97}
98
99static inline void debug_fence_free(struct i915_sw_fence *fence)
100{
101}
102
103static inline void debug_fence_assert(struct i915_sw_fence *fence)
104{
105}
106
107#endif
108
Chris Wilsone68a1392016-09-09 14:11:41 +0100109static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
110 enum i915_sw_fence_notify state)
111{
112 i915_sw_fence_notify_t fn;
113
114 fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
115 return fn(fence, state);
116}
117
Chris Wilsonfc158402016-11-25 13:17:18 +0000118#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
119void i915_sw_fence_fini(struct i915_sw_fence *fence)
120{
121 debug_fence_free(fence);
122}
123#endif
124
Chris Wilsone68a1392016-09-09 14:11:41 +0100125static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
126 struct list_head *continuation)
127{
128 wait_queue_head_t *x = &fence->wait;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200129 wait_queue_entry_t *pos, *next;
Chris Wilsone68a1392016-09-09 14:11:41 +0100130 unsigned long flags;
131
Chris Wilsonfc158402016-11-25 13:17:18 +0000132 debug_fence_deactivate(fence);
Chris Wilsone68a1392016-09-09 14:11:41 +0100133 atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
134
135 /*
136 * To prevent unbounded recursion as we traverse the graph of
Ingo Molnar2055da92017-06-20 12:06:46 +0200137 * i915_sw_fences, we move the entry list from this, the next ready
138 * fence, to the tail of the original fence's entry list
Chris Wilsone68a1392016-09-09 14:11:41 +0100139 * (and so added to the list to be woken).
140 */
141
142 spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
143 if (continuation) {
Ingo Molnar2055da92017-06-20 12:06:46 +0200144 list_for_each_entry_safe(pos, next, &x->head, entry) {
Chris Wilsone68a1392016-09-09 14:11:41 +0100145 if (pos->func == autoremove_wake_function)
146 pos->func(pos, TASK_NORMAL, 0, continuation);
147 else
Ingo Molnar2055da92017-06-20 12:06:46 +0200148 list_move_tail(&pos->entry, continuation);
Chris Wilsone68a1392016-09-09 14:11:41 +0100149 }
150 } else {
151 LIST_HEAD(extra);
152
153 do {
Ingo Molnar2055da92017-06-20 12:06:46 +0200154 list_for_each_entry_safe(pos, next, &x->head, entry)
Chris Wilsone68a1392016-09-09 14:11:41 +0100155 pos->func(pos, TASK_NORMAL, 0, &extra);
156
157 if (list_empty(&extra))
158 break;
159
Ingo Molnar2055da92017-06-20 12:06:46 +0200160 list_splice_tail_init(&extra, &x->head);
Chris Wilsone68a1392016-09-09 14:11:41 +0100161 } while (1);
162 }
163 spin_unlock_irqrestore(&x->lock, flags);
Chris Wilsonfc158402016-11-25 13:17:18 +0000164
165 debug_fence_assert(fence);
Chris Wilsone68a1392016-09-09 14:11:41 +0100166}
167
168static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
169 struct list_head *continuation)
170{
Chris Wilsonfc158402016-11-25 13:17:18 +0000171 debug_fence_assert(fence);
172
Chris Wilsone68a1392016-09-09 14:11:41 +0100173 if (!atomic_dec_and_test(&fence->pending))
174 return;
175
Chris Wilsonfc158402016-11-25 13:17:18 +0000176 debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
177
Chris Wilson9310cb72017-05-17 13:09:56 +0100178 if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
Chris Wilsone68a1392016-09-09 14:11:41 +0100179 return;
180
Chris Wilsonfc158402016-11-25 13:17:18 +0000181 debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
182
Chris Wilsone68a1392016-09-09 14:11:41 +0100183 __i915_sw_fence_wake_up_all(fence, continuation);
Chris Wilson9310cb72017-05-17 13:09:56 +0100184
185 debug_fence_destroy(fence);
186 __i915_sw_fence_notify(fence, FENCE_FREE);
Chris Wilsone68a1392016-09-09 14:11:41 +0100187}
188
189static void i915_sw_fence_complete(struct i915_sw_fence *fence)
190{
Chris Wilsonfc158402016-11-25 13:17:18 +0000191 debug_fence_assert(fence);
192
Chris Wilsone68a1392016-09-09 14:11:41 +0100193 if (WARN_ON(i915_sw_fence_done(fence)))
194 return;
195
196 __i915_sw_fence_complete(fence, NULL);
197}
198
199static void i915_sw_fence_await(struct i915_sw_fence *fence)
200{
Chris Wilsonfc158402016-11-25 13:17:18 +0000201 debug_fence_assert(fence);
Chris Wilsone68a1392016-09-09 14:11:41 +0100202 WARN_ON(atomic_inc_return(&fence->pending) <= 1);
203}
204
Chris Wilson556b7482016-11-14 20:40:56 +0000205void __i915_sw_fence_init(struct i915_sw_fence *fence,
206 i915_sw_fence_notify_t fn,
207 const char *name,
208 struct lock_class_key *key)
Chris Wilsone68a1392016-09-09 14:11:41 +0100209{
Chris Wilson9310cb72017-05-17 13:09:56 +0100210 BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
Chris Wilsone68a1392016-09-09 14:11:41 +0100211
Chris Wilsonfc158402016-11-25 13:17:18 +0000212 debug_fence_init(fence);
213
Chris Wilson556b7482016-11-14 20:40:56 +0000214 __init_waitqueue_head(&fence->wait, name, key);
Chris Wilsone68a1392016-09-09 14:11:41 +0100215 atomic_set(&fence->pending, 1);
216 fence->flags = (unsigned long)fn;
217}
218
Chris Wilsonfc158402016-11-25 13:17:18 +0000219void i915_sw_fence_commit(struct i915_sw_fence *fence)
220{
221 debug_fence_activate(fence);
Chris Wilson9310cb72017-05-17 13:09:56 +0100222 i915_sw_fence_complete(fence);
Chris Wilsonfc158402016-11-25 13:17:18 +0000223}
224
Ingo Molnarac6424b2017-06-20 12:06:13 +0200225static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
Chris Wilsone68a1392016-09-09 14:11:41 +0100226{
Ingo Molnar2055da92017-06-20 12:06:46 +0200227 list_del(&wq->entry);
Chris Wilsone68a1392016-09-09 14:11:41 +0100228 __i915_sw_fence_complete(wq->private, key);
Chris Wilson9310cb72017-05-17 13:09:56 +0100229
Chris Wilson7e941862016-10-28 13:58:25 +0100230 if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
231 kfree(wq);
Chris Wilsone68a1392016-09-09 14:11:41 +0100232 return 0;
233}
234
235static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
236 const struct i915_sw_fence * const signaler)
237{
Ingo Molnarac6424b2017-06-20 12:06:13 +0200238 wait_queue_entry_t *wq;
Chris Wilsone68a1392016-09-09 14:11:41 +0100239
240 if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
241 return false;
242
243 if (fence == signaler)
244 return true;
245
Ingo Molnar2055da92017-06-20 12:06:46 +0200246 list_for_each_entry(wq, &fence->wait.head, entry) {
Chris Wilsone68a1392016-09-09 14:11:41 +0100247 if (wq->func != i915_sw_fence_wake)
248 continue;
249
250 if (__i915_sw_fence_check_if_after(wq->private, signaler))
251 return true;
252 }
253
254 return false;
255}
256
257static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
258{
Ingo Molnarac6424b2017-06-20 12:06:13 +0200259 wait_queue_entry_t *wq;
Chris Wilsone68a1392016-09-09 14:11:41 +0100260
261 if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
262 return;
263
Ingo Molnar2055da92017-06-20 12:06:46 +0200264 list_for_each_entry(wq, &fence->wait.head, entry) {
Chris Wilsone68a1392016-09-09 14:11:41 +0100265 if (wq->func != i915_sw_fence_wake)
266 continue;
267
268 __i915_sw_fence_clear_checked_bit(wq->private);
269 }
270}
271
272static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
273 const struct i915_sw_fence * const signaler)
274{
275 unsigned long flags;
276 bool err;
277
Chris Wilson47624cc2017-05-17 13:09:57 +0100278 if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
Chris Wilsone68a1392016-09-09 14:11:41 +0100279 return false;
280
281 spin_lock_irqsave(&i915_sw_fence_lock, flags);
282 err = __i915_sw_fence_check_if_after(fence, signaler);
283 __i915_sw_fence_clear_checked_bit(fence);
284 spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
285
286 return err;
287}
288
Chris Wilson7e941862016-10-28 13:58:25 +0100289static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
290 struct i915_sw_fence *signaler,
Ingo Molnarac6424b2017-06-20 12:06:13 +0200291 wait_queue_entry_t *wq, gfp_t gfp)
Chris Wilsone68a1392016-09-09 14:11:41 +0100292{
293 unsigned long flags;
294 int pending;
295
Chris Wilsonfc158402016-11-25 13:17:18 +0000296 debug_fence_assert(fence);
297
Chris Wilsone68a1392016-09-09 14:11:41 +0100298 if (i915_sw_fence_done(signaler))
299 return 0;
300
Chris Wilsonfc158402016-11-25 13:17:18 +0000301 debug_fence_assert(signaler);
302
Chris Wilsone68a1392016-09-09 14:11:41 +0100303 /* The dependency graph must be acyclic. */
304 if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
305 return -EINVAL;
306
Chris Wilson7e941862016-10-28 13:58:25 +0100307 pending = 0;
308 if (!wq) {
309 wq = kmalloc(sizeof(*wq), gfp);
310 if (!wq) {
311 if (!gfpflags_allow_blocking(gfp))
312 return -ENOMEM;
313
314 i915_sw_fence_wait(signaler);
315 return 0;
316 }
317
318 pending |= I915_SW_FENCE_FLAG_ALLOC;
319 }
320
Ingo Molnar2055da92017-06-20 12:06:46 +0200321 INIT_LIST_HEAD(&wq->entry);
Chris Wilson7e941862016-10-28 13:58:25 +0100322 wq->flags = pending;
Chris Wilsone68a1392016-09-09 14:11:41 +0100323 wq->func = i915_sw_fence_wake;
Chris Wilson9310cb72017-05-17 13:09:56 +0100324 wq->private = fence;
Chris Wilsone68a1392016-09-09 14:11:41 +0100325
326 i915_sw_fence_await(fence);
327
328 spin_lock_irqsave(&signaler->wait.lock, flags);
329 if (likely(!i915_sw_fence_done(signaler))) {
Ingo Molnarac6424b2017-06-20 12:06:13 +0200330 __add_wait_queue_entry_tail(&signaler->wait, wq);
Chris Wilsone68a1392016-09-09 14:11:41 +0100331 pending = 1;
332 } else {
333 i915_sw_fence_wake(wq, 0, 0, NULL);
334 pending = 0;
335 }
336 spin_unlock_irqrestore(&signaler->wait.lock, flags);
337
338 return pending;
339}
340
Chris Wilson7e941862016-10-28 13:58:25 +0100341int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
342 struct i915_sw_fence *signaler,
Ingo Molnarac6424b2017-06-20 12:06:13 +0200343 wait_queue_entry_t *wq)
Chris Wilson7e941862016-10-28 13:58:25 +0100344{
345 return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
346}
347
348int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
349 struct i915_sw_fence *signaler,
350 gfp_t gfp)
351{
352 return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
353}
354
Chris Wilsonf54d1862016-10-25 13:00:45 +0100355struct i915_sw_dma_fence_cb {
356 struct dma_fence_cb base;
Chris Wilsone68a1392016-09-09 14:11:41 +0100357 struct i915_sw_fence *fence;
Chris Wilsonf54d1862016-10-25 13:00:45 +0100358 struct dma_fence *dma;
Chris Wilsone68a1392016-09-09 14:11:41 +0100359 struct timer_list timer;
Chris Wilson81c0ed22017-09-11 09:41:26 +0100360 struct irq_work work;
Chris Wilsone68a1392016-09-09 14:11:41 +0100361};
362
363static void timer_i915_sw_fence_wake(unsigned long data)
364{
Chris Wilsonf54d1862016-10-25 13:00:45 +0100365 struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
Chris Wilson81c0ed22017-09-11 09:41:26 +0100366 struct i915_sw_fence *fence;
367
368 fence = xchg(&cb->fence, NULL);
369 if (!fence)
370 return;
Chris Wilsone68a1392016-09-09 14:11:41 +0100371
Joe Perches8dfe1622017-02-28 04:55:54 -0800372 pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
373 cb->dma->ops->get_driver_name(cb->dma),
374 cb->dma->ops->get_timeline_name(cb->dma),
375 cb->dma->seqno);
Chris Wilsone68a1392016-09-09 14:11:41 +0100376
Chris Wilson81c0ed22017-09-11 09:41:26 +0100377 i915_sw_fence_complete(fence);
Chris Wilsone68a1392016-09-09 14:11:41 +0100378}
379
Chris Wilsonf54d1862016-10-25 13:00:45 +0100380static void dma_i915_sw_fence_wake(struct dma_fence *dma,
381 struct dma_fence_cb *data)
Chris Wilsone68a1392016-09-09 14:11:41 +0100382{
Chris Wilsonf54d1862016-10-25 13:00:45 +0100383 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
Chris Wilson81c0ed22017-09-11 09:41:26 +0100384 struct i915_sw_fence *fence;
385
386 fence = xchg(&cb->fence, NULL);
387 if (fence)
388 i915_sw_fence_complete(fence);
389
390 irq_work_queue(&cb->work);
391}
392
393static void irq_i915_sw_fence_work(struct irq_work *wrk)
394{
395 struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work);
Chris Wilsone68a1392016-09-09 14:11:41 +0100396
397 del_timer_sync(&cb->timer);
Chris Wilsonf54d1862016-10-25 13:00:45 +0100398 dma_fence_put(cb->dma);
Chris Wilsone68a1392016-09-09 14:11:41 +0100399
400 kfree(cb);
401}
402
403int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
Chris Wilsonf54d1862016-10-25 13:00:45 +0100404 struct dma_fence *dma,
Chris Wilsone68a1392016-09-09 14:11:41 +0100405 unsigned long timeout,
406 gfp_t gfp)
407{
Chris Wilsonf54d1862016-10-25 13:00:45 +0100408 struct i915_sw_dma_fence_cb *cb;
Chris Wilsone68a1392016-09-09 14:11:41 +0100409 int ret;
410
Chris Wilsonfc158402016-11-25 13:17:18 +0000411 debug_fence_assert(fence);
412
Chris Wilsonf54d1862016-10-25 13:00:45 +0100413 if (dma_fence_is_signaled(dma))
Chris Wilsone68a1392016-09-09 14:11:41 +0100414 return 0;
415
416 cb = kmalloc(sizeof(*cb), gfp);
417 if (!cb) {
418 if (!gfpflags_allow_blocking(gfp))
419 return -ENOMEM;
420
Chris Wilsonf54d1862016-10-25 13:00:45 +0100421 return dma_fence_wait(dma, false);
Chris Wilsone68a1392016-09-09 14:11:41 +0100422 }
423
Chris Wilson9310cb72017-05-17 13:09:56 +0100424 cb->fence = fence;
Chris Wilsone68a1392016-09-09 14:11:41 +0100425 i915_sw_fence_await(fence);
426
427 cb->dma = NULL;
428 __setup_timer(&cb->timer,
429 timer_i915_sw_fence_wake, (unsigned long)cb,
430 TIMER_IRQSAFE);
Chris Wilson81c0ed22017-09-11 09:41:26 +0100431 init_irq_work(&cb->work, irq_i915_sw_fence_work);
Chris Wilsone68a1392016-09-09 14:11:41 +0100432 if (timeout) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100433 cb->dma = dma_fence_get(dma);
Chris Wilsone68a1392016-09-09 14:11:41 +0100434 mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
435 }
436
Chris Wilsonf54d1862016-10-25 13:00:45 +0100437 ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
Chris Wilsone68a1392016-09-09 14:11:41 +0100438 if (ret == 0) {
439 ret = 1;
440 } else {
441 dma_i915_sw_fence_wake(dma, &cb->base);
442 if (ret == -ENOENT) /* fence already signaled */
443 ret = 0;
444 }
445
446 return ret;
447}
448
449int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
450 struct reservation_object *resv,
Chris Wilsonf54d1862016-10-25 13:00:45 +0100451 const struct dma_fence_ops *exclude,
Chris Wilsone68a1392016-09-09 14:11:41 +0100452 bool write,
453 unsigned long timeout,
454 gfp_t gfp)
455{
Chris Wilsonf54d1862016-10-25 13:00:45 +0100456 struct dma_fence *excl;
Chris Wilsone68a1392016-09-09 14:11:41 +0100457 int ret = 0, pending;
458
Chris Wilsonfc158402016-11-25 13:17:18 +0000459 debug_fence_assert(fence);
460
Chris Wilsone68a1392016-09-09 14:11:41 +0100461 if (write) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100462 struct dma_fence **shared;
Chris Wilsone68a1392016-09-09 14:11:41 +0100463 unsigned int count, i;
464
465 ret = reservation_object_get_fences_rcu(resv,
466 &excl, &count, &shared);
467 if (ret)
468 return ret;
469
470 for (i = 0; i < count; i++) {
471 if (shared[i]->ops == exclude)
472 continue;
473
474 pending = i915_sw_fence_await_dma_fence(fence,
475 shared[i],
476 timeout,
477 gfp);
478 if (pending < 0) {
479 ret = pending;
480 break;
481 }
482
483 ret |= pending;
484 }
485
486 for (i = 0; i < count; i++)
Chris Wilsonf54d1862016-10-25 13:00:45 +0100487 dma_fence_put(shared[i]);
Chris Wilsone68a1392016-09-09 14:11:41 +0100488 kfree(shared);
489 } else {
490 excl = reservation_object_get_excl_rcu(resv);
491 }
492
493 if (ret >= 0 && excl && excl->ops != exclude) {
494 pending = i915_sw_fence_await_dma_fence(fence,
495 excl,
496 timeout,
497 gfp);
498 if (pending < 0)
499 ret = pending;
500 else
501 ret |= pending;
502 }
503
Chris Wilsonf54d1862016-10-25 13:00:45 +0100504 dma_fence_put(excl);
Chris Wilsone68a1392016-09-09 14:11:41 +0100505
506 return ret;
507}
Chris Wilson47624cc2017-05-17 13:09:57 +0100508
509#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
510#include "selftests/i915_sw_fence.c"
511#endif