blob: c7d3236577984a0aab4c99b2e424a88a2504b386 [file] [log] [blame]
Thomas Hellstromc078aa22009-12-06 21:46:26 +01001/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/ttm/ttm_execbuf_util.h>
29#include <drm/ttm/ttm_bo_driver.h>
30#include <drm/ttm/ttm_placement.h>
Thomas Hellstromc078aa22009-12-06 21:46:26 +010031#include <linux/wait.h>
32#include <linux/sched.h>
33#include <linux/module.h>
34
Dave Airlied6ea8882010-11-22 13:24:40 +100035static void ttm_eu_backoff_reservation_locked(struct list_head *list)
36{
37 struct ttm_validate_buffer *entry;
38
39 list_for_each_entry(entry, list, head) {
40 struct ttm_buffer_object *bo = entry->bo;
41 if (!entry->reserved)
42 continue;
43
44 if (entry->removed) {
45 ttm_bo_add_to_lru(bo);
46 entry->removed = false;
47
48 }
49 entry->reserved = false;
50 atomic_set(&bo->reserved, 0);
51 wake_up_all(&bo->event_queue);
52 }
53}
54
55static void ttm_eu_del_from_lru_locked(struct list_head *list)
56{
57 struct ttm_validate_buffer *entry;
58
59 list_for_each_entry(entry, list, head) {
60 struct ttm_buffer_object *bo = entry->bo;
61 if (!entry->reserved)
62 continue;
63
64 if (!entry->removed) {
65 entry->put_count = ttm_bo_del_from_lru(bo);
66 entry->removed = true;
67 }
68 }
69}
70
71static void ttm_eu_list_ref_sub(struct list_head *list)
72{
73 struct ttm_validate_buffer *entry;
74
75 list_for_each_entry(entry, list, head) {
76 struct ttm_buffer_object *bo = entry->bo;
77
78 if (entry->put_count) {
79 ttm_bo_list_ref_sub(bo, entry->put_count, true);
80 entry->put_count = 0;
81 }
82 }
83}
84
Thomas Hellstromc078aa22009-12-06 21:46:26 +010085void ttm_eu_backoff_reservation(struct list_head *list)
86{
87 struct ttm_validate_buffer *entry;
Thomas Hellstrom68c4fa32010-11-17 12:28:27 +000088 struct ttm_bo_global *glob;
Thomas Hellstromc078aa22009-12-06 21:46:26 +010089
Thomas Hellstrom68c4fa32010-11-17 12:28:27 +000090 if (list_empty(list))
91 return;
Thomas Hellstromc078aa22009-12-06 21:46:26 +010092
Thomas Hellstrom68c4fa32010-11-17 12:28:27 +000093 entry = list_first_entry(list, struct ttm_validate_buffer, head);
94 glob = entry->bo->glob;
95 spin_lock(&glob->lru_lock);
96 ttm_eu_backoff_reservation_locked(list);
97 spin_unlock(&glob->lru_lock);
Thomas Hellstromc078aa22009-12-06 21:46:26 +010098}
99EXPORT_SYMBOL(ttm_eu_backoff_reservation);
100
101/*
102 * Reserve buffers for validation.
103 *
104 * If a buffer in the list is marked for CPU access, we back off and
105 * wait for that buffer to become free for GPU access.
106 *
107 * If a buffer is reserved for another validation, the validator with
108 * the highest validation sequence backs off and waits for that buffer
109 * to become unreserved. This prevents deadlocks when validating multiple
110 * buffers in different orders.
111 */
112
Thomas Hellstrom65705962010-11-17 12:28:31 +0000113int ttm_eu_reserve_buffers(struct list_head *list)
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100114{
Dave Airlied6ea8882010-11-22 13:24:40 +1000115 struct ttm_bo_global *glob;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100116 struct ttm_validate_buffer *entry;
117 int ret;
Thomas Hellstrom65705962010-11-17 12:28:31 +0000118 uint32_t val_seq;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100119
Dave Airlied6ea8882010-11-22 13:24:40 +1000120 if (list_empty(list))
121 return 0;
122
123 list_for_each_entry(entry, list, head) {
124 entry->reserved = false;
125 entry->put_count = 0;
126 entry->removed = false;
127 }
128
129 entry = list_first_entry(list, struct ttm_validate_buffer, head);
130 glob = entry->bo->glob;
131
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100132retry:
Dave Airlied6ea8882010-11-22 13:24:40 +1000133 spin_lock(&glob->lru_lock);
Thomas Hellstrom65705962010-11-17 12:28:31 +0000134 val_seq = entry->bo->bdev->val_seq++;
135
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100136 list_for_each_entry(entry, list, head) {
137 struct ttm_buffer_object *bo = entry->bo;
138
Maarten Lankhorst63d0a412013-01-15 14:56:37 +0100139 ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq);
Dave Airlied6ea8882010-11-22 13:24:40 +1000140 switch (ret) {
141 case 0:
142 break;
143 case -EBUSY:
Maarten Lankhorst7a186302013-01-15 14:56:48 +0100144 ttm_eu_del_from_lru_locked(list);
145 spin_unlock(&glob->lru_lock);
146 ret = ttm_bo_reserve_nolru(bo, true, false,
147 true, val_seq);
148 spin_lock(&glob->lru_lock);
149 if (!ret)
150 break;
151
152 if (unlikely(ret != -EAGAIN))
153 goto err;
154
155 /* fallthrough */
Dave Airlied6ea8882010-11-22 13:24:40 +1000156 case -EAGAIN:
157 ttm_eu_backoff_reservation_locked(list);
158 spin_unlock(&glob->lru_lock);
159 ttm_eu_list_ref_sub(list);
160 ret = ttm_bo_wait_unreserved(bo, true);
161 if (unlikely(ret != 0))
162 return ret;
163 goto retry;
164 default:
Maarten Lankhorst7a186302013-01-15 14:56:48 +0100165 goto err;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100166 }
167
168 entry->reserved = true;
169 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
Maarten Lankhorst7a186302013-01-15 14:56:48 +0100170 ret = -EBUSY;
171 goto err;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100172 }
173 }
Dave Airlied6ea8882010-11-22 13:24:40 +1000174
175 ttm_eu_del_from_lru_locked(list);
176 spin_unlock(&glob->lru_lock);
177 ttm_eu_list_ref_sub(list);
178
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100179 return 0;
Maarten Lankhorst7a186302013-01-15 14:56:48 +0100180
181err:
182 ttm_eu_backoff_reservation_locked(list);
183 spin_unlock(&glob->lru_lock);
184 ttm_eu_list_ref_sub(list);
185 return ret;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100186}
187EXPORT_SYMBOL(ttm_eu_reserve_buffers);
188
189void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
190{
191 struct ttm_validate_buffer *entry;
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000192 struct ttm_buffer_object *bo;
193 struct ttm_bo_global *glob;
194 struct ttm_bo_device *bdev;
195 struct ttm_bo_driver *driver;
196
197 if (list_empty(list))
198 return;
199
200 bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo;
201 bdev = bo->bdev;
202 driver = bdev->driver;
203 glob = bo->glob;
204
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000205 spin_lock(&glob->lru_lock);
Maarten Lankhorst4154f052012-11-28 12:25:39 +0100206 spin_lock(&bdev->fence_lock);
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100207
208 list_for_each_entry(entry, list, head) {
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000209 bo = entry->bo;
210 entry->old_sync_obj = bo->sync_obj;
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100211 bo->sync_obj = driver->sync_obj_ref(sync_obj);
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000212 ttm_bo_unreserve_locked(bo);
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100213 entry->reserved = false;
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000214 }
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000215 spin_unlock(&bdev->fence_lock);
Maarten Lankhorst4154f052012-11-28 12:25:39 +0100216 spin_unlock(&glob->lru_lock);
Thomas Hellstrom95762c22010-11-17 12:28:30 +0000217
218 list_for_each_entry(entry, list, head) {
219 if (entry->old_sync_obj)
220 driver->sync_obj_unref(&entry->old_sync_obj);
Thomas Hellstromc078aa22009-12-06 21:46:26 +0100221 }
222}
223EXPORT_SYMBOL(ttm_eu_fence_buffer_objects);