Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 1 | // |
| 2 | // Copyright 2012 Francisco Jerez |
| 3 | // |
| 4 | // Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | // copy of this software and associated documentation files (the "Software"), |
| 6 | // to deal in the Software without restriction, including without limitation |
| 7 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | // and/or sell copies of the Software, and to permit persons to whom the |
| 9 | // Software is furnished to do so, subject to the following conditions: |
| 10 | // |
| 11 | // The above copyright notice and this permission notice shall be included in |
| 12 | // all copies or substantial portions of the Software. |
| 13 | // |
| 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
Kenneth Graunke | f0cb66b | 2013-04-21 13:52:08 -0700 | [diff] [blame] | 17 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 | // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 | // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 | // OTHER DEALINGS IN THE SOFTWARE. |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 21 | // |
| 22 | |
| 23 | #include "core/event.hpp" |
| 24 | #include "pipe/p_screen.h" |
| 25 | |
| 26 | using namespace clover; |
| 27 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 28 | event::event(clover::context &ctx, const ref_vector<event> &deps, |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 29 | action action_ok, action action_fail) : |
Francisco Jerez | 02f8ac6 | 2015-06-09 22:52:25 +0300 | [diff] [blame] | 30 | context(ctx), _wait_count(1), _status(0), |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 31 | action_ok(action_ok), action_fail(action_fail) { |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 32 | for (auto &ev : deps) |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 33 | ev.chain(*this); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 34 | } |
| 35 | |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 36 | event::~event() { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 37 | } |
| 38 | |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 39 | std::vector<intrusive_ref<event>> |
| 40 | event::trigger_self() { |
Francisco Jerez | a533d4e | 2015-05-09 16:01:23 +0300 | [diff] [blame] | 41 | std::lock_guard<std::mutex> lock(mutex); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 42 | std::vector<intrusive_ref<event>> evs; |
| 43 | |
Jan Vesely | 1e8b8e0 | 2018-07-24 22:17:28 -0400 | [diff] [blame] | 44 | if (_wait_count && !--_wait_count) |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 45 | std::swap(_chain, evs); |
| 46 | |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 47 | cv.notify_all(); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 48 | return evs; |
| 49 | } |
| 50 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 51 | void |
Jan Vesely | 866b25f | 2018-07-17 02:05:02 -0400 | [diff] [blame] | 52 | event::trigger() try { |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 53 | if (wait_count() == 1) |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 54 | action_ok(*this); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 55 | |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 56 | for (event &ev : trigger_self()) |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 57 | ev.trigger(); |
Jan Vesely | 866b25f | 2018-07-17 02:05:02 -0400 | [diff] [blame] | 58 | } catch (error &e) { |
| 59 | abort(e.get()); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 60 | } |
| 61 | |
| 62 | std::vector<intrusive_ref<event>> |
| 63 | event::abort_self(cl_int status) { |
Francisco Jerez | a533d4e | 2015-05-09 16:01:23 +0300 | [diff] [blame] | 64 | std::lock_guard<std::mutex> lock(mutex); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 65 | std::vector<intrusive_ref<event>> evs; |
| 66 | |
| 67 | _status = status; |
Jan Vesely | 1e8b8e0 | 2018-07-24 22:17:28 -0400 | [diff] [blame] | 68 | _wait_count = 0; |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 69 | std::swap(_chain, evs); |
| 70 | |
Jan Vesely | 1e8b8e0 | 2018-07-24 22:17:28 -0400 | [diff] [blame] | 71 | cv.notify_all(); |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 72 | return evs; |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 73 | } |
| 74 | |
| 75 | void |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 76 | event::abort(cl_int status) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 77 | action_fail(*this); |
| 78 | |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 79 | for (event &ev : abort_self(status)) |
Francisco Jerez | 2232b92 | 2015-05-09 14:47:38 +0300 | [diff] [blame] | 80 | ev.abort(status); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 81 | } |
| 82 | |
Francisco Jerez | 02f8ac6 | 2015-06-09 22:52:25 +0300 | [diff] [blame] | 83 | unsigned |
| 84 | event::wait_count() const { |
| 85 | std::lock_guard<std::mutex> lock(mutex); |
| 86 | return _wait_count; |
| 87 | } |
| 88 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 89 | bool |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 90 | event::signalled() const { |
Francisco Jerez | 02f8ac6 | 2015-06-09 22:52:25 +0300 | [diff] [blame] | 91 | return !wait_count(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 92 | } |
| 93 | |
Francisco Jerez | 4022a46 | 2015-05-09 16:22:33 +0300 | [diff] [blame] | 94 | cl_int |
| 95 | event::status() const { |
Francisco Jerez | a533d4e | 2015-05-09 16:01:23 +0300 | [diff] [blame] | 96 | std::lock_guard<std::mutex> lock(mutex); |
Francisco Jerez | 4022a46 | 2015-05-09 16:22:33 +0300 | [diff] [blame] | 97 | return _status; |
| 98 | } |
| 99 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 100 | void |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 101 | event::chain(event &ev) { |
Francisco Jerez | a533d4e | 2015-05-09 16:01:23 +0300 | [diff] [blame] | 102 | std::unique_lock<std::mutex> lock(mutex, std::defer_lock); |
| 103 | std::unique_lock<std::mutex> lock_ev(ev.mutex, std::defer_lock); |
| 104 | std::lock(lock, lock_ev); |
| 105 | |
Francisco Jerez | 02f8ac6 | 2015-06-09 22:52:25 +0300 | [diff] [blame] | 106 | if (_wait_count) { |
| 107 | ev._wait_count++; |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 108 | _chain.push_back(ev); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 109 | } |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 110 | ev.deps.push_back(*this); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 111 | } |
| 112 | |
Tom Stellard | 9c4dc98 | 2015-03-26 19:33:24 +0000 | [diff] [blame] | 113 | void |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 114 | event::wait_signalled() const { |
| 115 | std::unique_lock<std::mutex> lock(mutex); |
| 116 | cv.wait(lock, [=]{ return !_wait_count; }); |
| 117 | } |
| 118 | |
| 119 | void |
Tom Stellard | 9c4dc98 | 2015-03-26 19:33:24 +0000 | [diff] [blame] | 120 | event::wait() const { |
| 121 | for (event &ev : deps) |
| 122 | ev.wait(); |
| 123 | |
Francisco Jerez | bc4000e | 2015-06-09 22:59:43 +0300 | [diff] [blame] | 124 | wait_signalled(); |
Tom Stellard | 9c4dc98 | 2015-03-26 19:33:24 +0000 | [diff] [blame] | 125 | } |
| 126 | |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 127 | hard_event::hard_event(command_queue &q, cl_command_type command, |
| 128 | const ref_vector<event> &deps, action action) : |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 129 | event(q.context(), deps, profile(q, action), [](event &ev){}), |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 130 | _queue(q), _command(command), _fence(NULL) { |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 131 | if (q.profiling_enabled()) |
| 132 | _time_queued = timestamp::current(q); |
| 133 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 134 | q.sequence(*this); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 135 | trigger(); |
| 136 | } |
| 137 | |
| 138 | hard_event::~hard_event() { |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 139 | pipe_screen *screen = queue()->device().pipe; |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 140 | screen->fence_reference(screen, &_fence, NULL); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 141 | } |
| 142 | |
| 143 | cl_int |
| 144 | hard_event::status() const { |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 145 | pipe_screen *screen = queue()->device().pipe; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 146 | |
Francisco Jerez | 4022a46 | 2015-05-09 16:22:33 +0300 | [diff] [blame] | 147 | if (event::status() < 0) |
| 148 | return event::status(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 149 | |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 150 | else if (!_fence) |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 151 | return CL_QUEUED; |
| 152 | |
Marek Olšák | 54272e1 | 2016-08-06 16:41:42 +0200 | [diff] [blame] | 153 | else if (!screen->fence_finish(screen, NULL, _fence, 0)) |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 154 | return CL_SUBMITTED; |
| 155 | |
| 156 | else |
| 157 | return CL_COMPLETE; |
| 158 | } |
| 159 | |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 160 | command_queue * |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 161 | hard_event::queue() const { |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 162 | return &_queue(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | cl_command_type |
| 166 | hard_event::command() const { |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 167 | return _command; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | void |
| 171 | hard_event::wait() const { |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 172 | pipe_screen *screen = queue()->device().pipe; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 173 | |
Tom Stellard | 9c4dc98 | 2015-03-26 19:33:24 +0000 | [diff] [blame] | 174 | event::wait(); |
| 175 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 176 | if (status() == CL_QUEUED) |
| 177 | queue()->flush(); |
| 178 | |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 179 | if (!_fence || |
Marek Olšák | 54272e1 | 2016-08-06 16:41:42 +0200 | [diff] [blame] | 180 | !screen->fence_finish(screen, NULL, _fence, PIPE_TIMEOUT_INFINITE)) |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 181 | throw error(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST); |
| 182 | } |
| 183 | |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 184 | const lazy<cl_ulong> & |
| 185 | hard_event::time_queued() const { |
| 186 | return _time_queued; |
Niels Ole Salscheider | 4a3505d | 2013-08-09 11:59:25 +0200 | [diff] [blame] | 187 | } |
| 188 | |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 189 | const lazy<cl_ulong> & |
| 190 | hard_event::time_submit() const { |
| 191 | return _time_submit; |
Niels Ole Salscheider | 4a3505d | 2013-08-09 11:59:25 +0200 | [diff] [blame] | 192 | } |
| 193 | |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 194 | const lazy<cl_ulong> & |
| 195 | hard_event::time_start() const { |
| 196 | return _time_start; |
Niels Ole Salscheider | 4a3505d | 2013-08-09 11:59:25 +0200 | [diff] [blame] | 197 | } |
| 198 | |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 199 | const lazy<cl_ulong> & |
| 200 | hard_event::time_end() const { |
| 201 | return _time_end; |
Niels Ole Salscheider | 4a3505d | 2013-08-09 11:59:25 +0200 | [diff] [blame] | 202 | } |
| 203 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 204 | void |
| 205 | hard_event::fence(pipe_fence_handle *fence) { |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 206 | pipe_screen *screen = queue()->device().pipe; |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 207 | screen->fence_reference(screen, &_fence, fence); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 208 | } |
| 209 | |
Francisco Jerez | ebfdce0 | 2013-10-06 13:48:23 -0700 | [diff] [blame] | 210 | event::action |
| 211 | hard_event::profile(command_queue &q, const action &action) const { |
| 212 | if (q.profiling_enabled()) { |
| 213 | return [&q, action] (event &ev) { |
| 214 | auto &hev = static_cast<hard_event &>(ev); |
| 215 | |
| 216 | hev._time_submit = timestamp::current(q); |
| 217 | hev._time_start = timestamp::query(q); |
| 218 | |
| 219 | action(ev); |
| 220 | |
| 221 | hev._time_end = timestamp::query(q); |
| 222 | }; |
| 223 | |
| 224 | } else { |
| 225 | return action; |
| 226 | } |
| 227 | } |
| 228 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 229 | soft_event::soft_event(clover::context &ctx, const ref_vector<event> &deps, |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 230 | bool _trigger, action action) : |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 231 | event(ctx, deps, action, action) { |
Francisco Jerez | 8e14b82 | 2013-09-17 23:13:48 -0700 | [diff] [blame] | 232 | if (_trigger) |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 233 | trigger(); |
| 234 | } |
| 235 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 236 | cl_int |
| 237 | soft_event::status() const { |
Francisco Jerez | 4022a46 | 2015-05-09 16:22:33 +0300 | [diff] [blame] | 238 | if (event::status() < 0) |
| 239 | return event::status(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 240 | |
| 241 | else if (!signalled() || |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 242 | any_of([](const event &ev) { |
| 243 | return ev.status() != CL_COMPLETE; |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 244 | }, deps)) |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 245 | return CL_SUBMITTED; |
| 246 | |
| 247 | else |
| 248 | return CL_COMPLETE; |
| 249 | } |
| 250 | |
Francisco Jerez | 257781f | 2013-10-01 11:54:07 -0700 | [diff] [blame] | 251 | command_queue * |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 252 | soft_event::queue() const { |
| 253 | return NULL; |
| 254 | } |
| 255 | |
| 256 | cl_command_type |
| 257 | soft_event::command() const { |
| 258 | return CL_COMMAND_USER; |
| 259 | } |
| 260 | |
| 261 | void |
| 262 | soft_event::wait() const { |
Tom Stellard | 9c4dc98 | 2015-03-26 19:33:24 +0000 | [diff] [blame] | 263 | event::wait(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 264 | |
| 265 | if (status() != CL_COMPLETE) |
| 266 | throw error(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST); |
| 267 | } |