clover: implements clEnqueueMigrateMemObjects
Reviewed-by: Francisco Jerez <currojerez@riseup.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4974>
diff --git a/src/gallium/frontends/clover/api/transfer.cpp b/src/gallium/frontends/clover/api/transfer.cpp
index aa21b00..b2964b2 100644
--- a/src/gallium/frontends/clover/api/transfer.cpp
+++ b/src/gallium/frontends/clover/api/transfer.cpp
@@ -210,7 +210,7 @@
struct _map<image*> {
_map(command_queue &q, image *img, cl_map_flags flags,
vector_t offset, vector_t pitch, vector_t region) :
- map(q, img->resource(q), flags, true, offset, region),
+ map(q, img->resource_in(q), flags, true, offset, region),
pitch(map.pitch())
{ }
@@ -227,7 +227,7 @@
struct _map<buffer*> {
_map(command_queue &q, buffer *mem, cl_map_flags flags,
vector_t offset, vector_t pitch, vector_t region) :
- map(q, mem->resource(q), flags, true,
+ map(q, mem->resource_in(q), flags, true,
{{ dot(pitch, offset) }}, {{ size(pitch, region) }}),
pitch(pitch)
{ }
@@ -294,8 +294,8 @@
hard_copy_op(command_queue &q, T dst_obj, const vector_t &dst_orig,
S src_obj, const vector_t &src_orig, const vector_t ®ion) {
return [=, &q](event &) {
- dst_obj->resource(q).copy(q, dst_orig, region,
- src_obj->resource(q), src_orig);
+ dst_obj->resource_in(q).copy(q, dst_orig, region,
+ src_obj->resource_in(q), src_orig);
};
}
}
@@ -480,7 +480,7 @@
auto hev = create<hard_event>(
q, CL_COMMAND_FILL_BUFFER, deps,
[=, &q, &mem](event &) {
- mem.resource(q).clear(q, offset, size, &data[0], data.size());
+ mem.resource_in(q).clear(q, offset, size, &data[0], data.size());
});
ret_object(rd_ev, hev);
@@ -755,7 +755,7 @@
validate_object(q, mem, obj_origin, obj_pitch, region);
validate_map_flags(mem, flags);
- void *map = mem.resource(q).add_map(q, flags, blocking, obj_origin, region);
+ void *map = mem.resource_in(q).add_map(q, flags, blocking, obj_origin, region);
auto hev = create<hard_event>(q, CL_COMMAND_MAP_BUFFER, deps);
if (blocking)
@@ -787,7 +787,7 @@
validate_object(q, img, origin, region);
validate_map_flags(img, flags);
- void *map = img.resource(q).add_map(q, flags, blocking, origin, region);
+ void *map = img.resource_in(q).add_map(q, flags, blocking, origin, region);
auto hev = create<hard_event>(q, CL_COMMAND_MAP_IMAGE, deps);
if (blocking)
@@ -815,7 +815,7 @@
auto hev = create<hard_event>(
q, CL_COMMAND_UNMAP_MEM_OBJECT, deps,
[=, &q, &mem](event &) {
- mem.resource(q).del_map(ptr);
+ mem.resource_in(q).del_map(ptr);
});
ret_object(rd_ev, hev);
@@ -826,15 +826,54 @@
}
CLOVER_API cl_int
-clEnqueueMigrateMemObjects(cl_command_queue command_queue,
- cl_uint num_mem_objects,
- const cl_mem *mem_objects,
+clEnqueueMigrateMemObjects(cl_command_queue d_q,
+ cl_uint num_mems,
+ const cl_mem *d_mems,
cl_mem_migration_flags flags,
- cl_uint num_events_in_wait_list,
- const cl_event *event_wait_list,
- cl_event *event) {
- CLOVER_NOT_SUPPORTED_UNTIL("1.2");
- return CL_INVALID_VALUE;
+ cl_uint num_deps,
+ const cl_event *d_deps,
+ cl_event *rd_ev) try {
+ auto &q = obj(d_q);
+ auto mems = objs<memory_obj>(d_mems, num_mems);
+ auto deps = objs<wait_list_tag>(d_deps, num_deps);
+
+ validate_common(q, deps);
+
+ if (any_of([&](const memory_obj &m) {
+ return m.context() != q.context();
+ }, mems))
+ throw error(CL_INVALID_CONTEXT);
+
+ if (flags & ~(CL_MIGRATE_MEM_OBJECT_HOST |
+ CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED))
+ throw error(CL_INVALID_VALUE);
+
+ auto hev = create<hard_event>(
+ q, CL_COMMAND_MIGRATE_MEM_OBJECTS, deps,
+ [=, &q](event &) {
+ for (auto &mem: mems) {
+ if (flags & CL_MIGRATE_MEM_OBJECT_HOST) {
+ if ((flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED))
+ mem.resource_out(q);
+
+ // For flags == CL_MIGRATE_MEM_OBJECT_HOST only to be
+ // efficient we would need cl*ReadBuffer* to implement
+ // reading from host memory.
+
+ } else {
+ if (flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED)
+ mem.resource_undef(q);
+ else
+ mem.resource_in(q);
+ }
+ }
+ });
+
+ ret_object(rd_ev, hev);
+ return CL_SUCCESS;;
+
+} catch (error &e) {
+ return e.get();
}
cl_int
diff --git a/src/gallium/frontends/clover/core/kernel.cpp b/src/gallium/frontends/clover/core/kernel.cpp
index 013d6f0..3000b7c 100644
--- a/src/gallium/frontends/clover/core/kernel.cpp
+++ b/src/gallium/frontends/clover/core/kernel.cpp
@@ -447,7 +447,7 @@
align(ctx.input, marg.target_align);
if (buf) {
- const resource &r = buf->resource(*ctx.q);
+ const resource &r = buf->resource_in(*ctx.q);
ctx.g_handles.push_back(ctx.input.size());
ctx.g_buffers.push_back(r.pipe);
@@ -522,7 +522,7 @@
align(ctx.input, marg.target_align);
if (buf) {
- resource &r = buf->resource(*ctx.q);
+ resource &r = buf->resource_in(*ctx.q);
auto v = bytes(ctx.resources.size() << 24 | r.offset[0]);
extend(v, module::argument::zero_ext, marg.target_size);
@@ -540,7 +540,7 @@
void
kernel::constant_argument::unbind(exec_context &ctx) {
if (buf)
- buf->resource(*ctx.q).unbind_surface(*ctx.q, st);
+ buf->resource_in(*ctx.q).unbind_surface(*ctx.q, st);
}
void
@@ -565,13 +565,13 @@
align(ctx.input, marg.target_align);
insert(ctx.input, v);
- st = img->resource(*ctx.q).bind_sampler_view(*ctx.q);
+ st = img->resource_in(*ctx.q).bind_sampler_view(*ctx.q);
ctx.sviews.push_back(st);
}
void
kernel::image_rd_argument::unbind(exec_context &ctx) {
- img->resource(*ctx.q).unbind_sampler_view(*ctx.q, st);
+ img->resource_in(*ctx.q).unbind_sampler_view(*ctx.q, st);
}
void
@@ -596,13 +596,13 @@
align(ctx.input, marg.target_align);
insert(ctx.input, v);
- st = img->resource(*ctx.q).bind_surface(*ctx.q, true);
+ st = img->resource_in(*ctx.q).bind_surface(*ctx.q, true);
ctx.resources.push_back(st);
}
void
kernel::image_wr_argument::unbind(exec_context &ctx) {
- img->resource(*ctx.q).unbind_surface(*ctx.q, st);
+ img->resource_in(*ctx.q).unbind_surface(*ctx.q, st);
}
void
diff --git a/src/gallium/frontends/clover/core/memory.cpp b/src/gallium/frontends/clover/core/memory.cpp
index ed13d92..d20dbad 100644
--- a/src/gallium/frontends/clover/core/memory.cpp
+++ b/src/gallium/frontends/clover/core/memory.cpp
@@ -82,13 +82,27 @@
}
resource &
-root_buffer::resource(command_queue &q) {
+root_buffer::resource_in(command_queue &q) {
+ const void *data_ptr = NULL;
+ if (flags() & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR))
+ data_ptr = !data.empty() ? data.data() : host_ptr();
+
+ return resource(q, data_ptr);
+}
+
+resource &
+root_buffer::resource_undef(command_queue &q) {
+ return resource(q, NULL);
+}
+
+resource &
+root_buffer::resource(command_queue &q, const void *data_ptr) {
// Create a new resource if there's none for this device yet.
if (!resources.count(&q.device())) {
auto r = (!resources.empty() ?
new root_resource(q.device(), *this,
*resources.begin()->second) :
- new root_resource(q.device(), *this, q, data));
+ new root_resource(q.device(), *this, q, data_ptr));
resources.insert(std::make_pair(&q.device(),
std::unique_ptr<root_resource>(r)));
@@ -98,6 +112,11 @@
return *resources.find(&q.device())->second;
}
+void
+root_buffer::resource_out(command_queue &q) {
+ resources.erase(&q.device());
+}
+
sub_buffer::sub_buffer(root_buffer &parent, cl_mem_flags flags,
size_t offset, size_t size) :
buffer(parent.context(), flags, size,
@@ -106,10 +125,10 @@
}
resource &
-sub_buffer::resource(command_queue &q) {
+sub_buffer::resource_in(command_queue &q) {
// Create a new resource if there's none for this device yet.
if (!resources.count(&q.device())) {
- auto r = new sub_resource(parent().resource(q), {{ offset() }});
+ auto r = new sub_resource(parent().resource_in(q), {{ offset() }});
resources.insert(std::make_pair(&q.device(),
std::unique_ptr<sub_resource>(r)));
@@ -118,6 +137,16 @@
return *resources.find(&q.device())->second;
}
+resource &
+sub_buffer::resource_undef(command_queue &q) {
+ return resource_in(q);
+}
+
+void
+sub_buffer::resource_out(command_queue &q) {
+ resources.erase(&q.device());
+}
+
size_t
sub_buffer::offset() const {
return _offset;
@@ -134,13 +163,24 @@
}
resource &
-image::resource(command_queue &q) {
+image::resource_in(command_queue &q) {
+ const void *data_ptr = !data.empty() ? data.data() : NULL;
+ return resource(q, data_ptr);
+}
+
+resource &
+image::resource_undef(command_queue &q) {
+ return resource(q, NULL);
+}
+
+resource &
+image::resource(command_queue &q, const void *data_ptr) {
// Create a new resource if there's none for this device yet.
if (!resources.count(&q.device())) {
auto r = (!resources.empty() ?
new root_resource(q.device(), *this,
*resources.begin()->second) :
- new root_resource(q.device(), *this, q, data));
+ new root_resource(q.device(), *this, q, data_ptr));
resources.insert(std::make_pair(&q.device(),
std::unique_ptr<root_resource>(r)));
@@ -150,6 +190,11 @@
return *resources.find(&q.device())->second;
}
+void
+image::resource_out(command_queue &q) {
+ resources.erase(&q.device());
+}
+
cl_image_format
image::format() const {
return _format;
diff --git a/src/gallium/frontends/clover/core/memory.hpp b/src/gallium/frontends/clover/core/memory.hpp
index bd6da6b..7c48110 100644
--- a/src/gallium/frontends/clover/core/memory.hpp
+++ b/src/gallium/frontends/clover/core/memory.hpp
@@ -49,7 +49,11 @@
operator==(const memory_obj &obj) const;
virtual cl_mem_object_type type() const = 0;
- virtual clover::resource &resource(command_queue &q) = 0;
+ virtual clover::resource &
+ resource_in(command_queue &q) = 0;
+ virtual clover::resource &
+ resource_undef(command_queue &q) = 0;
+ virtual void resource_out(command_queue &q) = 0;
void destroy_notify(std::function<void ()> f);
cl_mem_flags flags() const;
@@ -82,9 +86,17 @@
root_buffer(clover::context &ctx, cl_mem_flags flags,
size_t size, void *host_ptr);
- virtual clover::resource &resource(command_queue &q);
+ virtual clover::resource &
+ resource_in(command_queue &q);
+ virtual clover::resource &
+ resource_undef(command_queue &q);
+ virtual void
+ resource_out(command_queue &q);
private:
+ clover::resource &
+ resource(command_queue &q, const void *data_ptr);
+
std::map<device *,
std::unique_ptr<root_resource>> resources;
};
@@ -94,7 +106,12 @@
sub_buffer(root_buffer &parent, cl_mem_flags flags,
size_t offset, size_t size);
- virtual clover::resource &resource(command_queue &q);
+ virtual clover::resource &
+ resource_in(command_queue &q);
+ virtual clover::resource &
+ resource_undef(command_queue &q);
+ virtual void
+ resource_out(command_queue &q);
size_t offset() const;
const intrusive_ref<root_buffer> parent;
@@ -114,7 +131,6 @@
void *host_ptr);
public:
- virtual clover::resource &resource(command_queue &q);
cl_image_format format() const;
size_t width() const;
size_t height() const;
@@ -122,8 +138,17 @@
size_t pixel_size() const;
size_t row_pitch() const;
size_t slice_pitch() const;
+ virtual clover::resource &
+ resource_in(command_queue &q);
+ virtual clover::resource &
+ resource_undef(command_queue &q);
+ virtual void
+ resource_out(command_queue &q);
private:
+ clover::resource &
+ resource(command_queue &q, const void *data_ptr);
+
cl_image_format _format;
size_t _width;
size_t _height;
diff --git a/src/gallium/frontends/clover/core/resource.cpp b/src/gallium/frontends/clover/core/resource.cpp
index 2fbed90..f43163d 100644
--- a/src/gallium/frontends/clover/core/resource.cpp
+++ b/src/gallium/frontends/clover/core/resource.cpp
@@ -124,7 +124,7 @@
}
root_resource::root_resource(clover::device &dev, memory_obj &obj,
- command_queue &q, const std::string &data) :
+ command_queue &q, const void *data_ptr) :
resource(dev, obj) {
pipe_resource info {};
@@ -161,8 +161,7 @@
if (!pipe)
throw error(CL_OUT_OF_RESOURCES);
- if (obj.flags() & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR)) {
- const void *data_ptr = !data.empty() ? data.data() : obj.host_ptr();
+ if (data_ptr) {
box rect { {{ 0, 0, 0 }}, {{ info.width0, info.height0, info.depth0 }} };
unsigned cpp = util_format_get_blocksize(info.format);
diff --git a/src/gallium/frontends/clover/core/resource.hpp b/src/gallium/frontends/clover/core/resource.hpp
index 009e42f..51ecc53 100644
--- a/src/gallium/frontends/clover/core/resource.hpp
+++ b/src/gallium/frontends/clover/core/resource.hpp
@@ -89,7 +89,7 @@
class root_resource : public resource {
public:
root_resource(clover::device &dev, memory_obj &obj,
- command_queue &q, const std::string &data);
+ command_queue &q, const void *data_ptr);
root_resource(clover::device &dev, memory_obj &obj, root_resource &r);
virtual ~root_resource();
};