Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 1 | // |
| 2 | // Copyright 2012 Francisco Jerez |
| 3 | // |
| 4 | // Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | // copy of this software and associated documentation files (the "Software"), |
| 6 | // to deal in the Software without restriction, including without limitation |
| 7 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | // and/or sell copies of the Software, and to permit persons to whom the |
| 9 | // Software is furnished to do so, subject to the following conditions: |
| 10 | // |
| 11 | // The above copyright notice and this permission notice shall be included in |
| 12 | // all copies or substantial portions of the Software. |
| 13 | // |
| 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
Kenneth Graunke | f0cb66b | 2013-04-21 13:52:08 -0700 | [diff] [blame] | 17 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 | // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 | // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 | // OTHER DEALINGS IN THE SOFTWARE. |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 21 | // |
| 22 | |
| 23 | #include "core/resource.hpp" |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 24 | #include "core/memory.hpp" |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 25 | #include "pipe/p_screen.h" |
| 26 | #include "util/u_sampler.h" |
Eric Anholt | 882ca6d | 2019-06-27 15:05:31 -0700 | [diff] [blame] | 27 | #include "util/format/u_format.h" |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 28 | #include "util/u_inlines.h" |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 29 | |
| 30 | using namespace clover; |
| 31 | |
| 32 | namespace { |
| 33 | class box { |
| 34 | public: |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 35 | box(const resource::vector &origin, const resource::vector &size) : |
Aaron Watry | 4d0399f | 2017-04-04 20:16:02 -0500 | [diff] [blame] | 36 | pipe({ (int)origin[0], (int16_t)origin[1], |
| 37 | (int16_t)origin[2], (int)size[0], |
| 38 | (int16_t)size[1], (int16_t)size[2] }) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 39 | } |
| 40 | |
| 41 | operator const pipe_box *() { |
| 42 | return &pipe; |
| 43 | } |
| 44 | |
| 45 | protected: |
| 46 | pipe_box pipe; |
| 47 | }; |
| 48 | } |
| 49 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 50 | resource::resource(clover::device &dev, memory_obj &obj) : |
| 51 | device(dev), obj(obj), pipe(NULL), offset() { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 52 | } |
| 53 | |
| 54 | resource::~resource() { |
| 55 | } |
| 56 | |
| 57 | void |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 58 | resource::copy(command_queue &q, const vector &origin, const vector ®ion, |
| 59 | resource &src_res, const vector &src_origin) { |
| 60 | auto p = offset + origin; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 61 | |
| 62 | q.pipe->resource_copy_region(q.pipe, pipe, 0, p[0], p[1], p[2], |
| 63 | src_res.pipe, 0, |
| 64 | box(src_res.offset + src_origin, region)); |
| 65 | } |
| 66 | |
Serge Martin | dad042b | 2020-05-03 13:56:15 +0200 | [diff] [blame] | 67 | void |
Serge Martin | d7d66e3 | 2020-05-09 12:29:39 +0200 | [diff] [blame] | 68 | resource::clear(command_queue &q, const vector &origin, const vector ®ion, |
| 69 | const std::string &data) { |
| 70 | auto from = offset + origin; |
Serge Martin | dad042b | 2020-05-03 13:56:15 +0200 | [diff] [blame] | 71 | |
Serge Martin | d7d66e3 | 2020-05-09 12:29:39 +0200 | [diff] [blame] | 72 | if (pipe->target == PIPE_BUFFER) { |
| 73 | q.pipe->clear_buffer(q.pipe, pipe, from[0], region[0], data.data(), data.size()); |
| 74 | } else { |
| 75 | std::string texture_data; |
| 76 | texture_data.reserve(util_format_get_blocksize(pipe->format)); |
| 77 | util_format_pack_rgba(pipe->format, &texture_data[0], data.data(), 1); |
| 78 | q.pipe->clear_texture(q.pipe, pipe, 0, box(from, region), texture_data.data()); |
| 79 | } |
Serge Martin | dad042b | 2020-05-03 13:56:15 +0200 | [diff] [blame] | 80 | } |
| 81 | |
Karol Herbst | f2bdb69 | 2020-10-15 01:19:57 +0200 | [diff] [blame] | 82 | mapping * |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 83 | resource::add_map(command_queue &q, cl_map_flags flags, bool blocking, |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 84 | const vector &origin, const vector ®ion) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 85 | maps.emplace_back(q, *this, flags, blocking, origin, region); |
Karol Herbst | f2bdb69 | 2020-10-15 01:19:57 +0200 | [diff] [blame] | 86 | return &maps.back(); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | void |
| 90 | resource::del_map(void *p) { |
Francisco Jerez | 1d741e3 | 2013-09-15 23:07:10 -0700 | [diff] [blame] | 91 | erase_if([&](const mapping &m) { |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 92 | return static_cast<void *>(m) == p; |
| 93 | }, maps); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 94 | } |
| 95 | |
| 96 | unsigned |
| 97 | resource::map_count() const { |
| 98 | return maps.size(); |
| 99 | } |
| 100 | |
| 101 | pipe_sampler_view * |
Francisco Jerez | d6f7afc | 2013-10-01 12:00:51 -0700 | [diff] [blame] | 102 | resource::bind_sampler_view(command_queue &q) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 103 | pipe_sampler_view info; |
| 104 | |
| 105 | u_sampler_view_default_template(&info, pipe, pipe->format); |
| 106 | return q.pipe->create_sampler_view(q.pipe, pipe, &info); |
| 107 | } |
| 108 | |
| 109 | void |
Francisco Jerez | d6f7afc | 2013-10-01 12:00:51 -0700 | [diff] [blame] | 110 | resource::unbind_sampler_view(command_queue &q, |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 111 | pipe_sampler_view *st) { |
| 112 | q.pipe->sampler_view_destroy(q.pipe, st); |
| 113 | } |
| 114 | |
Karol Herbst | 3aead71 | 2020-10-07 23:08:43 +0200 | [diff] [blame] | 115 | pipe_image_view |
| 116 | resource::create_image_view(command_queue &q) { |
| 117 | pipe_image_view view; |
| 118 | view.resource = pipe; |
| 119 | view.format = pipe->format; |
| 120 | view.access = 0; |
| 121 | view.shader_access = PIPE_IMAGE_ACCESS_WRITE; |
| 122 | |
| 123 | if (pipe->target == PIPE_BUFFER) { |
| 124 | view.u.buf.offset = 0; |
| 125 | view.u.buf.size = obj.size(); |
| 126 | } else { |
| 127 | view.u.tex.first_layer = 0; |
| 128 | view.u.tex.last_layer = 0; |
| 129 | view.u.tex.level = 0; |
| 130 | } |
| 131 | |
| 132 | return view; |
| 133 | } |
| 134 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 135 | pipe_surface * |
Francisco Jerez | d6f7afc | 2013-10-01 12:00:51 -0700 | [diff] [blame] | 136 | resource::bind_surface(command_queue &q, bool rw) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 137 | pipe_surface info {}; |
| 138 | |
| 139 | info.format = pipe->format; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 140 | info.writable = rw; |
| 141 | |
| 142 | if (pipe->target == PIPE_BUFFER) |
| 143 | info.u.buf.last_element = pipe->width0 - 1; |
| 144 | |
| 145 | return q.pipe->create_surface(q.pipe, pipe, &info); |
| 146 | } |
| 147 | |
| 148 | void |
Francisco Jerez | d6f7afc | 2013-10-01 12:00:51 -0700 | [diff] [blame] | 149 | resource::unbind_surface(command_queue &q, pipe_surface *st) { |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 150 | q.pipe->surface_destroy(q.pipe, st); |
| 151 | } |
| 152 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 153 | root_resource::root_resource(clover::device &dev, memory_obj &obj, |
Serge Martin | c0f03f6 | 2020-05-09 08:11:16 +0200 | [diff] [blame] | 154 | command_queue &q, const void *data_ptr) : |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 155 | resource(dev, obj) { |
| 156 | pipe_resource info {}; |
| 157 | |
| 158 | if (image *img = dynamic_cast<image *>(&obj)) { |
| 159 | info.format = translate_format(img->format()); |
| 160 | info.width0 = img->width(); |
| 161 | info.height0 = img->height(); |
| 162 | info.depth0 = img->depth(); |
| 163 | } else { |
| 164 | info.width0 = obj.size(); |
Francisco Jerez | 60e7b08 | 2012-05-04 15:02:21 +0200 | [diff] [blame] | 165 | info.height0 = 1; |
| 166 | info.depth0 = 1; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 167 | } |
| 168 | |
Zoltan Gilian | aa46fba | 2015-07-27 11:27:12 +0200 | [diff] [blame] | 169 | info.array_size = 1; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 170 | info.target = translate_target(obj.type()); |
| 171 | info.bind = (PIPE_BIND_SAMPLER_VIEW | |
| 172 | PIPE_BIND_COMPUTE_RESOURCE | |
Marek Olšák | 5981ab5 | 2016-09-07 21:24:08 +0200 | [diff] [blame] | 173 | PIPE_BIND_GLOBAL); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 174 | |
Karol Herbst | c0f7f83 | 2020-05-05 15:09:50 +0200 | [diff] [blame] | 175 | if (obj.flags() & CL_MEM_USE_HOST_PTR && dev.allows_user_pointers()) { |
Grigori Goronzy | f972b22 | 2015-05-19 09:28:30 +0200 | [diff] [blame] | 176 | // Page alignment is normally required for this, just try, hope for the |
| 177 | // best and fall back if it fails. |
| 178 | pipe = dev.pipe->resource_from_user_memory(dev.pipe, &info, obj.host_ptr()); |
| 179 | if (pipe) |
| 180 | return; |
| 181 | } |
| 182 | |
| 183 | if (obj.flags() & (CL_MEM_ALLOC_HOST_PTR | CL_MEM_USE_HOST_PTR)) { |
Grigori Goronzy | 5c495e8 | 2015-05-12 02:22:12 +0200 | [diff] [blame] | 184 | info.usage = PIPE_USAGE_STAGING; |
| 185 | } |
| 186 | |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 187 | pipe = dev.pipe->resource_create(dev.pipe, &info); |
| 188 | if (!pipe) |
| 189 | throw error(CL_OUT_OF_RESOURCES); |
| 190 | |
Serge Martin | c0f03f6 | 2020-05-09 08:11:16 +0200 | [diff] [blame] | 191 | if (data_ptr) { |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 192 | box rect { {{ 0, 0, 0 }}, {{ info.width0, info.height0, info.depth0 }} }; |
Francisco Jerez | 60e7b08 | 2012-05-04 15:02:21 +0200 | [diff] [blame] | 193 | unsigned cpp = util_format_get_blocksize(info.format); |
| 194 | |
Marek Olšák | 1ffe77e | 2016-07-16 21:19:48 +0200 | [diff] [blame] | 195 | if (pipe->target == PIPE_BUFFER) |
Marek Olšák | 22253e6 | 2020-07-01 08:16:12 -0400 | [diff] [blame] | 196 | q.pipe->buffer_subdata(q.pipe, pipe, PIPE_MAP_WRITE, |
Marek Olšák | 1ffe77e | 2016-07-16 21:19:48 +0200 | [diff] [blame] | 197 | 0, info.width0, data_ptr); |
| 198 | else |
Marek Olšák | 22253e6 | 2020-07-01 08:16:12 -0400 | [diff] [blame] | 199 | q.pipe->texture_subdata(q.pipe, pipe, 0, PIPE_MAP_WRITE, |
Marek Olšák | 1ffe77e | 2016-07-16 21:19:48 +0200 | [diff] [blame] | 200 | rect, data_ptr, cpp * info.width0, |
| 201 | cpp * info.width0 * info.height0); |
Francisco Jerez | 60e7b08 | 2012-05-04 15:02:21 +0200 | [diff] [blame] | 202 | } |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 203 | } |
| 204 | |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 205 | root_resource::root_resource(clover::device &dev, memory_obj &obj, |
Francisco Jerez | d6f7afc | 2013-10-01 12:00:51 -0700 | [diff] [blame] | 206 | root_resource &r) : |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 207 | resource(dev, obj) { |
| 208 | assert(0); // XXX -- resource shared among dev and r.dev |
| 209 | } |
| 210 | |
| 211 | root_resource::~root_resource() { |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 212 | pipe_resource_reference(&this->pipe, NULL); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 213 | } |
| 214 | |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 215 | sub_resource::sub_resource(resource &r, const vector &offset) : |
Francisco Jerez | c4578d2 | 2014-02-18 15:07:11 +0100 | [diff] [blame] | 216 | resource(r.device(), r.obj) { |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 217 | this->pipe = r.pipe; |
| 218 | this->offset = r.offset + offset; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 219 | } |
| 220 | |
| 221 | mapping::mapping(command_queue &q, resource &r, |
| 222 | cl_map_flags flags, bool blocking, |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 223 | const resource::vector &origin, |
| 224 | const resource::vector ®ion) : |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 225 | pctx(q.pipe), pres(NULL) { |
Marek Olšák | 22253e6 | 2020-07-01 08:16:12 -0400 | [diff] [blame] | 226 | unsigned usage = ((flags & CL_MAP_WRITE ? PIPE_MAP_WRITE : 0 ) | |
| 227 | (flags & CL_MAP_READ ? PIPE_MAP_READ : 0 ) | |
Bruno Jiménez | ec73778 | 2014-08-07 16:34:53 +0200 | [diff] [blame] | 228 | (flags & CL_MAP_WRITE_INVALIDATE_REGION ? |
Marek Olšák | 22253e6 | 2020-07-01 08:16:12 -0400 | [diff] [blame] | 229 | PIPE_MAP_DISCARD_RANGE : 0) | |
| 230 | (!blocking ? PIPE_MAP_UNSYNCHRONIZED : 0)); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 231 | |
Marek Olšák | 369e468 | 2012-10-08 04:06:42 +0200 | [diff] [blame] | 232 | p = pctx->transfer_map(pctx, r.pipe, 0, usage, |
| 233 | box(origin + r.offset, region), &pxfer); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 234 | if (!p) { |
Marek Olšák | 369e468 | 2012-10-08 04:06:42 +0200 | [diff] [blame] | 235 | pxfer = NULL; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 236 | throw error(CL_OUT_OF_RESOURCES); |
| 237 | } |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 238 | pipe_resource_reference(&pres, r.pipe); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 239 | } |
| 240 | |
| 241 | mapping::mapping(mapping &&m) : |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 242 | pctx(m.pctx), pxfer(m.pxfer), pres(m.pres), p(m.p) { |
Francisco Jerez | 1d741e3 | 2013-09-15 23:07:10 -0700 | [diff] [blame] | 243 | m.pctx = NULL; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 244 | m.pxfer = NULL; |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 245 | m.pres = NULL; |
Francisco Jerez | 1d741e3 | 2013-09-15 23:07:10 -0700 | [diff] [blame] | 246 | m.p = NULL; |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 247 | } |
| 248 | |
| 249 | mapping::~mapping() { |
| 250 | if (pxfer) { |
| 251 | pctx->transfer_unmap(pctx, pxfer); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 252 | } |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 253 | pipe_resource_reference(&pres, NULL); |
Francisco Jerez | c6db1b3 | 2012-04-20 16:56:19 +0200 | [diff] [blame] | 254 | } |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 255 | |
| 256 | mapping & |
| 257 | mapping::operator=(mapping m) { |
| 258 | std::swap(pctx, m.pctx); |
| 259 | std::swap(pxfer, m.pxfer); |
Jan Vesely | 14b543b | 2017-03-01 19:45:03 -0500 | [diff] [blame] | 260 | std::swap(pres, m.pres); |
Francisco Jerez | 7d61769 | 2013-10-06 13:49:05 -0700 | [diff] [blame] | 261 | std::swap(p, m.p); |
| 262 | return *this; |
| 263 | } |
Jason Ekstrand | afa5c2e | 2020-08-31 16:43:40 -0500 | [diff] [blame] | 264 | |
| 265 | resource::vector |
| 266 | mapping::pitch() const |
| 267 | { |
| 268 | return { |
| 269 | util_format_get_blocksize(pres->format), |
| 270 | pxfer->stride, |
| 271 | pxfer->layer_stride, |
| 272 | }; |
| 273 | } |