blob: 015d07f4b04db8e1a4cd3aefa1738ac413230b25 [file] [log] [blame]
Shrenuj Bansala419c792016-10-20 14:05:11 -07001/* Copyright (c) 2012-2017, The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
13
14#include <linux/err.h>
15#include <linux/file.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18#include <linux/uaccess.h>
19
20#include <asm/current.h>
21
22#include "kgsl_sync.h"
23
Lynus Vazc031a9b2017-01-25 13:00:13 +053024static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *timeline,
Shrenuj Bansala419c792016-10-20 14:05:11 -070025 unsigned int timestamp);
26
Lynus Vazc031a9b2017-01-25 13:00:13 +053027static const struct fence_ops kgsl_sync_fence_ops;
28
29static struct kgsl_sync_fence *kgsl_sync_fence_create(
30 struct kgsl_context *context,
31 unsigned int timestamp)
Shrenuj Bansala419c792016-10-20 14:05:11 -070032{
Lynus Vazc031a9b2017-01-25 13:00:13 +053033 struct kgsl_sync_fence *kfence;
34 struct kgsl_sync_timeline *ktimeline = context->ktimeline;
35 unsigned long flags;
Shrenuj Bansala419c792016-10-20 14:05:11 -070036
Lynus Vazc031a9b2017-01-25 13:00:13 +053037 /* Get a refcount to the timeline. Put when released */
38 if (!kref_get_unless_zero(&ktimeline->kref))
39 return NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -070040
Lynus Vazc031a9b2017-01-25 13:00:13 +053041 kfence = kzalloc(sizeof(*kfence), GFP_KERNEL);
42 if (kfence == NULL) {
43 kgsl_sync_timeline_put(ktimeline);
44 KGSL_DRV_ERR(context->device, "Couldn't allocate fence\n");
45 return NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -070046 }
Lynus Vazc031a9b2017-01-25 13:00:13 +053047
48 kfence->parent = ktimeline;
49 kfence->context_id = context->id;
50 kfence->timestamp = timestamp;
51
52 fence_init(&kfence->fence, &kgsl_sync_fence_ops, &ktimeline->lock,
53 ktimeline->fence_context, timestamp);
54
Lynus Vaz9d6334b2017-06-27 14:57:56 +053055 /*
56 * sync_file_create() takes a refcount to the fence. This refcount is
57 * put when the fence is signaled.
58 */
Lynus Vazc031a9b2017-01-25 13:00:13 +053059 kfence->sync_file = sync_file_create(&kfence->fence);
60
61 if (kfence->sync_file == NULL) {
62 kgsl_sync_timeline_put(ktimeline);
63 KGSL_DRV_ERR(context->device, "Create sync_file failed\n");
64 kfree(kfence);
65 return NULL;
66 }
67
Lynus Vazc031a9b2017-01-25 13:00:13 +053068 spin_lock_irqsave(&ktimeline->lock, flags);
69 list_add_tail(&kfence->child_list, &ktimeline->child_list_head);
70 spin_unlock_irqrestore(&ktimeline->lock, flags);
71
72 return kfence;
Shrenuj Bansala419c792016-10-20 14:05:11 -070073}
74
Lynus Vazc031a9b2017-01-25 13:00:13 +053075static void kgsl_sync_fence_release(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070076{
Lynus Vazc031a9b2017-01-25 13:00:13 +053077 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
78
79 kgsl_sync_timeline_put(kfence->parent);
80 kfree(kfence);
Shrenuj Bansala419c792016-10-20 14:05:11 -070081}
82
Lynus Vazc031a9b2017-01-25 13:00:13 +053083/* Called with ktimeline->lock held */
84bool kgsl_sync_fence_has_signaled(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070085{
Lynus Vazc031a9b2017-01-25 13:00:13 +053086 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
87 struct kgsl_sync_timeline *ktimeline = kfence->parent;
88 unsigned int ts = kfence->timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -070089
Lynus Vazc031a9b2017-01-25 13:00:13 +053090 return (timestamp_cmp(ktimeline->last_timestamp, ts) >= 0);
Shrenuj Bansala419c792016-10-20 14:05:11 -070091}
92
Lynus Vazc031a9b2017-01-25 13:00:13 +053093bool kgsl_enable_signaling(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070094{
Lynus Vazc031a9b2017-01-25 13:00:13 +053095 return !kgsl_sync_fence_has_signaled(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -070096}
97
Lynus Vazc031a9b2017-01-25 13:00:13 +053098struct kgsl_sync_fence_event_priv {
Shrenuj Bansala419c792016-10-20 14:05:11 -070099 struct kgsl_context *context;
100 unsigned int timestamp;
101};
102
103/**
Lynus Vazc031a9b2017-01-25 13:00:13 +0530104 * kgsl_sync_fence_event_cb - Event callback for a fence timestamp event
Shrenuj Bansala419c792016-10-20 14:05:11 -0700105 * @device - The KGSL device that expired the timestamp
106 * @context- Pointer to the context that owns the event
107 * @priv: Private data for the callback
108 * @result - Result of the event (retired or canceled)
109 *
110 * Signal a fence following the expiration of a timestamp
111 */
112
Lynus Vazc031a9b2017-01-25 13:00:13 +0530113static void kgsl_sync_fence_event_cb(struct kgsl_device *device,
Shrenuj Bansala419c792016-10-20 14:05:11 -0700114 struct kgsl_event_group *group, void *priv, int result)
115{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530116 struct kgsl_sync_fence_event_priv *ev = priv;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700117
Lynus Vazc031a9b2017-01-25 13:00:13 +0530118 kgsl_sync_timeline_signal(ev->context->ktimeline, ev->timestamp);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700119 kgsl_context_put(ev->context);
120 kfree(ev);
121}
122
123static int _add_fence_event(struct kgsl_device *device,
124 struct kgsl_context *context, unsigned int timestamp)
125{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530126 struct kgsl_sync_fence_event_priv *event;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700127 int ret;
128
129 event = kmalloc(sizeof(*event), GFP_KERNEL);
130 if (event == NULL)
131 return -ENOMEM;
132
133 /*
134 * Increase the refcount for the context to keep it through the
135 * callback
136 */
137 if (!_kgsl_context_get(context)) {
138 kfree(event);
139 return -ENOENT;
140 }
141
142 event->context = context;
143 event->timestamp = timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700144
145 ret = kgsl_add_event(device, &context->events, timestamp,
Lynus Vazc031a9b2017-01-25 13:00:13 +0530146 kgsl_sync_fence_event_cb, event);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700147
148 if (ret) {
149 kgsl_context_put(context);
150 kfree(event);
151 }
152
153 return ret;
154}
155
Lynus Vazc50d6202017-03-23 13:42:58 +0530156/* Only to be used if creating a related event failed */
157static void kgsl_sync_cancel(struct kgsl_sync_fence *kfence)
158{
159 spin_lock(&kfence->parent->lock);
160 if (!list_empty(&kfence->child_list)) {
161 list_del_init(&kfence->child_list);
162 fence_put(&kfence->fence);
163 }
164 spin_unlock(&kfence->parent->lock);
165}
166
Shrenuj Bansala419c792016-10-20 14:05:11 -0700167/**
168 * kgsl_add_fence_event - Create a new fence event
169 * @device - KGSL device to create the event on
170 * @timestamp - Timestamp to trigger the event
171 * @data - Return fence fd stored in struct kgsl_timestamp_event_fence
172 * @len - length of the fence event
173 * @owner - driver instance that owns this event
174 * @returns 0 on success or error code on error
175 *
176 * Create a fence and register an event to signal the fence when
177 * the timestamp expires
178 */
179
180int kgsl_add_fence_event(struct kgsl_device *device,
181 u32 context_id, u32 timestamp, void __user *data, int len,
182 struct kgsl_device_private *owner)
183{
184 struct kgsl_timestamp_event_fence priv;
185 struct kgsl_context *context;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530186 struct kgsl_sync_fence *kfence = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700187 int ret = -EINVAL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700188 unsigned int cur;
189
190 priv.fence_fd = -1;
191
192 if (len != sizeof(priv))
193 return -EINVAL;
194
195 context = kgsl_context_get_owner(owner, context_id);
196
197 if (context == NULL)
198 return -EINVAL;
199
200 if (test_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv))
201 goto out;
202
Lynus Vazc031a9b2017-01-25 13:00:13 +0530203 kfence = kgsl_sync_fence_create(context, timestamp);
204 if (kfence == NULL) {
205 KGSL_DRV_CRIT_RATELIMIT(device,
206 "kgsl_sync_fence_create failed\n");
Shrenuj Bansala419c792016-10-20 14:05:11 -0700207 ret = -ENOMEM;
208 goto out;
209 }
210
211 priv.fence_fd = get_unused_fd_flags(0);
212 if (priv.fence_fd < 0) {
213 KGSL_DRV_CRIT_RATELIMIT(device,
214 "Unable to get a file descriptor: %d\n",
215 priv.fence_fd);
216 ret = priv.fence_fd;
217 goto out;
218 }
219
220 /*
221 * If the timestamp hasn't expired yet create an event to trigger it.
222 * Otherwise, just signal the fence - there is no reason to go through
223 * the effort of creating a fence we don't need.
224 */
225
226 kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &cur);
227
228 if (timestamp_cmp(cur, timestamp) >= 0) {
229 ret = 0;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530230 kgsl_sync_timeline_signal(context->ktimeline, cur);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700231 } else {
232 ret = _add_fence_event(device, context, timestamp);
233 if (ret)
234 goto out;
235 }
236
237 if (copy_to_user(data, &priv, sizeof(priv))) {
238 ret = -EFAULT;
239 goto out;
240 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530241 fd_install(priv.fence_fd, kfence->sync_file->file);
242
Shrenuj Bansala419c792016-10-20 14:05:11 -0700243out:
244 kgsl_context_put(context);
245 if (ret) {
246 if (priv.fence_fd >= 0)
247 put_unused_fd(priv.fence_fd);
248
Lynus Vazc031a9b2017-01-25 13:00:13 +0530249 if (kfence) {
Lynus Vazc50d6202017-03-23 13:42:58 +0530250 kgsl_sync_cancel(kfence);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530251 /*
252 * Put the refcount of sync file. This will release
253 * kfence->fence as well.
254 */
255 fput(kfence->sync_file->file);
256 }
Shrenuj Bansala419c792016-10-20 14:05:11 -0700257 }
258 return ret;
259}
260
Lynus Vazc031a9b2017-01-25 13:00:13 +0530261static unsigned int kgsl_sync_fence_get_timestamp(
262 struct kgsl_sync_timeline *ktimeline,
263 enum kgsl_timestamp_type type)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700264{
265 unsigned int ret = 0;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700266
267 if (ktimeline->device == NULL)
268 return 0;
269
Lynus Vazc031a9b2017-01-25 13:00:13 +0530270 kgsl_readtimestamp(ktimeline->device, ktimeline->context, type, &ret);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700271
Shrenuj Bansala419c792016-10-20 14:05:11 -0700272 return ret;
273}
274
Lynus Vazc031a9b2017-01-25 13:00:13 +0530275static void kgsl_sync_timeline_value_str(struct fence *fence,
276 char *str, int size)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700277{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530278 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
279 struct kgsl_sync_timeline *ktimeline = kfence->parent;
280
281 unsigned int timestamp_retired;
282 unsigned int timestamp_queued;
283
284 if (!kref_get_unless_zero(&ktimeline->kref))
285 return;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700286
287 /*
288 * This callback can be called before the device and spinlock are
289 * initialized in struct kgsl_sync_timeline. kgsl_sync_get_timestamp()
290 * will check if device is NULL and return 0. Queued and retired
291 * timestamp of the context will be reported as 0, which is correct
292 * because the context and timeline are just getting initialized.
293 */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530294 timestamp_retired = kgsl_sync_fence_get_timestamp(ktimeline,
295 KGSL_TIMESTAMP_RETIRED);
296 timestamp_queued = kgsl_sync_fence_get_timestamp(ktimeline,
297 KGSL_TIMESTAMP_QUEUED);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700298
299 snprintf(str, size, "%u queued:%u retired:%u",
300 ktimeline->last_timestamp,
301 timestamp_queued, timestamp_retired);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530302
303 kgsl_sync_timeline_put(ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700304}
305
Lynus Vazc031a9b2017-01-25 13:00:13 +0530306static void kgsl_sync_fence_value_str(struct fence *fence, char *str, int size)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700307{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530308 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700309
Lynus Vazc031a9b2017-01-25 13:00:13 +0530310 snprintf(str, size, "%u", kfence->timestamp);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700311}
312
Lynus Vazc031a9b2017-01-25 13:00:13 +0530313static const char *kgsl_sync_fence_driver_name(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700314{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530315 return "kgsl-timeline";
Shrenuj Bansala419c792016-10-20 14:05:11 -0700316}
317
Lynus Vazc031a9b2017-01-25 13:00:13 +0530318static const char *kgsl_sync_timeline_name(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700319{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530320 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
321 struct kgsl_sync_timeline *ktimeline = kfence->parent;
322
323 return ktimeline->name;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700324}
Shrenuj Bansala419c792016-10-20 14:05:11 -0700325
326int kgsl_sync_timeline_create(struct kgsl_context *context)
327{
328 struct kgsl_sync_timeline *ktimeline;
329
330 /*
331 * Generate a name which includes the thread name, thread id, process
332 * name, process id, and context id. This makes it possible to
333 * identify the context of a timeline in the sync dump.
334 */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530335 char ktimeline_name[sizeof(ktimeline->name)] = {};
336
337 /* Put context when timeline is released */
338 if (!_kgsl_context_get(context))
339 return -ENOENT;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700340
341 snprintf(ktimeline_name, sizeof(ktimeline_name),
342 "%s_%.15s(%d)-%.15s(%d)-%d",
343 context->device->name,
344 current->group_leader->comm, current->group_leader->pid,
345 current->comm, current->pid, context->id);
346
Lynus Vazc031a9b2017-01-25 13:00:13 +0530347 ktimeline = kzalloc(sizeof(*ktimeline), GFP_KERNEL);
348 if (ktimeline == NULL) {
349 kgsl_context_put(context);
350 return -ENOMEM;
351 }
Shrenuj Bansala419c792016-10-20 14:05:11 -0700352
Lynus Vazc031a9b2017-01-25 13:00:13 +0530353 kref_init(&ktimeline->kref);
354 strlcpy(ktimeline->name, ktimeline_name, KGSL_TIMELINE_NAME_LEN);
355 ktimeline->fence_context = fence_context_alloc(1);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700356 ktimeline->last_timestamp = 0;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530357 INIT_LIST_HEAD(&ktimeline->child_list_head);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700358 spin_lock_init(&ktimeline->lock);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530359 ktimeline->device = context->device;
360 ktimeline->context = context;
361
362 context->ktimeline = ktimeline;
363
Shrenuj Bansala419c792016-10-20 14:05:11 -0700364 return 0;
365}
366
Lynus Vazc031a9b2017-01-25 13:00:13 +0530367static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *ktimeline,
368 unsigned int timestamp)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700369{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530370 unsigned long flags;
371 struct kgsl_sync_fence *kfence, *next;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700372
Lynus Vaz8795cc62017-07-28 21:33:11 +0530373 if (!kref_get_unless_zero(&ktimeline->kref))
374 return;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530375
376 spin_lock_irqsave(&ktimeline->lock, flags);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700377 if (timestamp_cmp(timestamp, ktimeline->last_timestamp) > 0)
378 ktimeline->last_timestamp = timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700379
Lynus Vazc031a9b2017-01-25 13:00:13 +0530380 list_for_each_entry_safe(kfence, next, &ktimeline->child_list_head,
381 child_list) {
382 if (fence_is_signaled_locked(&kfence->fence)) {
Lynus Vazc50d6202017-03-23 13:42:58 +0530383 list_del_init(&kfence->child_list);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530384 fence_put(&kfence->fence);
385 }
386 }
387
388 spin_unlock_irqrestore(&ktimeline->lock, flags);
389 kgsl_sync_timeline_put(ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700390}
391
392void kgsl_sync_timeline_destroy(struct kgsl_context *context)
393{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530394 kfree(context->ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700395}
396
Lynus Vazc031a9b2017-01-25 13:00:13 +0530397static void kgsl_sync_timeline_release(struct kref *kref)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700398{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530399 struct kgsl_sync_timeline *ktimeline =
400 container_of(kref, struct kgsl_sync_timeline, kref);
401
402 /*
403 * Only put the context refcount here. The context destroy function
404 * will call kgsl_sync_timeline_destroy() to kfree it
405 */
406 kgsl_context_put(ktimeline->context);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700407}
408
Lynus Vazc031a9b2017-01-25 13:00:13 +0530409void kgsl_sync_timeline_put(struct kgsl_sync_timeline *ktimeline)
410{
411 if (ktimeline)
412 kref_put(&ktimeline->kref, kgsl_sync_timeline_release);
413}
414
415static const struct fence_ops kgsl_sync_fence_ops = {
416 .get_driver_name = kgsl_sync_fence_driver_name,
417 .get_timeline_name = kgsl_sync_timeline_name,
418 .enable_signaling = kgsl_enable_signaling,
419 .signaled = kgsl_sync_fence_has_signaled,
420 .wait = fence_default_wait,
421 .release = kgsl_sync_fence_release,
422
423 .fence_value_str = kgsl_sync_fence_value_str,
424 .timeline_value_str = kgsl_sync_timeline_value_str,
425};
426
427static void kgsl_sync_fence_callback(struct fence *fence, struct fence_cb *cb)
428{
429 struct kgsl_sync_fence_cb *kcb = (struct kgsl_sync_fence_cb *)cb;
430
Lynus Vaz27da44d2017-07-26 13:50:10 +0530431 /*
432 * If the callback is marked for cancellation in a separate thread,
433 * let the other thread do the cleanup.
434 */
435 if (kcb->func(kcb->priv)) {
436 fence_put(kcb->fence);
437 kfree(kcb);
438 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530439}
440
Lynus Vaze99b92b2017-04-24 18:04:54 +0530441static void kgsl_get_fence_name(struct fence *fence,
442 char *fence_name, int name_len)
443{
444 char *ptr = fence_name;
445 char *last = fence_name + name_len;
446
447 ptr += snprintf(ptr, last - ptr, "%s %s",
448 fence->ops->get_driver_name(fence),
449 fence->ops->get_timeline_name(fence));
450
451 if ((ptr + 2) >= last)
452 return;
453
454 if (fence->ops->fence_value_str) {
455 ptr += snprintf(ptr, last - ptr, ": ");
456 fence->ops->fence_value_str(fence, ptr, last - ptr);
457 }
458}
459
Lynus Vazc031a9b2017-01-25 13:00:13 +0530460struct kgsl_sync_fence_cb *kgsl_sync_fence_async_wait(int fd,
Lynus Vaz27da44d2017-07-26 13:50:10 +0530461 bool (*func)(void *priv), void *priv, char *fence_name, int name_len)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700462{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530463 struct kgsl_sync_fence_cb *kcb;
464 struct fence *fence;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700465 int status;
466
Lynus Vazc031a9b2017-01-25 13:00:13 +0530467 fence = sync_file_get_fence(fd);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700468 if (fence == NULL)
469 return ERR_PTR(-EINVAL);
470
Lynus Vazc031a9b2017-01-25 13:00:13 +0530471 /* create the callback */
472 kcb = kzalloc(sizeof(*kcb), GFP_ATOMIC);
473 if (kcb == NULL) {
474 fence_put(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700475 return ERR_PTR(-ENOMEM);
476 }
477
Lynus Vazc031a9b2017-01-25 13:00:13 +0530478 kcb->fence = fence;
479 kcb->priv = priv;
480 kcb->func = func;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700481
Lynus Vaze99b92b2017-04-24 18:04:54 +0530482 if (fence_name)
483 kgsl_get_fence_name(fence, fence_name, name_len);
484
Shrenuj Bansala419c792016-10-20 14:05:11 -0700485 /* if status then error or signaled */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530486 status = fence_add_callback(fence, &kcb->fence_cb,
487 kgsl_sync_fence_callback);
488
Shrenuj Bansala419c792016-10-20 14:05:11 -0700489 if (status) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530490 kfree(kcb);
Harshdeep Dhatt0ed60a42017-04-20 16:41:50 -0600491 if (!fence_is_signaled(fence))
Lynus Vazc031a9b2017-01-25 13:00:13 +0530492 kcb = ERR_PTR(status);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700493 else
Lynus Vazc031a9b2017-01-25 13:00:13 +0530494 kcb = NULL;
495 fence_put(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700496 }
497
Lynus Vazc031a9b2017-01-25 13:00:13 +0530498 return kcb;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700499}
500
Lynus Vaz27da44d2017-07-26 13:50:10 +0530501/*
502 * Cancel the fence async callback and do the cleanup. The caller must make
503 * sure that the callback (if run before cancelling) returns false, so that
504 * no other thread frees the pointer.
505 */
506void kgsl_sync_fence_async_cancel(struct kgsl_sync_fence_cb *kcb)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700507{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530508 if (kcb == NULL)
Lynus Vaz27da44d2017-07-26 13:50:10 +0530509 return;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700510
Lynus Vaz27da44d2017-07-26 13:50:10 +0530511 /*
512 * After fence_remove_callback() returns, the fence callback is
513 * either not called at all, or completed without freeing kcb.
514 * This thread can then put the fence refcount and free kcb.
515 */
516 fence_remove_callback(kcb->fence, &kcb->fence_cb);
517 fence_put(kcb->fence);
518 kfree(kcb);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700519}
520
Shrenuj Bansala419c792016-10-20 14:05:11 -0700521struct kgsl_syncsource {
522 struct kref refcount;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530523 char name[KGSL_TIMELINE_NAME_LEN];
Shrenuj Bansala419c792016-10-20 14:05:11 -0700524 int id;
525 struct kgsl_process_private *private;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530526 struct list_head child_list_head;
527 spinlock_t lock;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700528};
529
Lynus Vazc031a9b2017-01-25 13:00:13 +0530530struct kgsl_syncsource_fence {
531 struct fence fence;
532 struct kgsl_syncsource *parent;
533 struct list_head child_list;
534};
535
536static const struct fence_ops kgsl_syncsource_fence_ops;
537
Shrenuj Bansala419c792016-10-20 14:05:11 -0700538long kgsl_ioctl_syncsource_create(struct kgsl_device_private *dev_priv,
539 unsigned int cmd, void *data)
540{
541 struct kgsl_syncsource *syncsource = NULL;
542 struct kgsl_syncsource_create *param = data;
543 int ret = -EINVAL;
544 int id = 0;
545 struct kgsl_process_private *private = dev_priv->process_priv;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530546 char name[KGSL_TIMELINE_NAME_LEN];
547
548 if (!kgsl_process_private_get(private))
549 return ret;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700550
551 syncsource = kzalloc(sizeof(*syncsource), GFP_KERNEL);
552 if (syncsource == NULL) {
553 ret = -ENOMEM;
554 goto out;
555 }
556
557 snprintf(name, sizeof(name), "kgsl-syncsource-pid-%d",
558 current->group_leader->pid);
559
Shrenuj Bansala419c792016-10-20 14:05:11 -0700560 kref_init(&syncsource->refcount);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530561 strlcpy(syncsource->name, name, KGSL_TIMELINE_NAME_LEN);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700562 syncsource->private = private;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530563 INIT_LIST_HEAD(&syncsource->child_list_head);
564 spin_lock_init(&syncsource->lock);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700565
566 idr_preload(GFP_KERNEL);
567 spin_lock(&private->syncsource_lock);
568 id = idr_alloc(&private->syncsource_idr, syncsource, 1, 0, GFP_NOWAIT);
569 if (id > 0) {
570 syncsource->id = id;
571 param->id = id;
572 ret = 0;
573 } else {
574 ret = id;
575 }
576
577 spin_unlock(&private->syncsource_lock);
578 idr_preload_end();
579
580out:
581 if (ret) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530582 kgsl_process_private_put(private);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700583 kfree(syncsource);
584 }
585
586 return ret;
587}
588
589static struct kgsl_syncsource *
590kgsl_syncsource_get(struct kgsl_process_private *private, int id)
591{
592 int result = 0;
593 struct kgsl_syncsource *syncsource = NULL;
594
595 spin_lock(&private->syncsource_lock);
596
597 syncsource = idr_find(&private->syncsource_idr, id);
598 if (syncsource)
599 result = kref_get_unless_zero(&syncsource->refcount);
600
601 spin_unlock(&private->syncsource_lock);
602
603 return result ? syncsource : NULL;
604}
605
606static void kgsl_syncsource_destroy(struct kref *kref)
607{
608 struct kgsl_syncsource *syncsource = container_of(kref,
609 struct kgsl_syncsource,
610 refcount);
611
612 struct kgsl_process_private *private = syncsource->private;
613
Lynus Vazc031a9b2017-01-25 13:00:13 +0530614 /* Done with process private. Release the refcount */
615 kgsl_process_private_put(private);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700616
617 kfree(syncsource);
618}
619
620void kgsl_syncsource_put(struct kgsl_syncsource *syncsource)
621{
622 if (syncsource)
623 kref_put(&syncsource->refcount, kgsl_syncsource_destroy);
624}
625
Lynus Vazc031a9b2017-01-25 13:00:13 +0530626void kgsl_syncsource_cleanup(struct kgsl_process_private *private,
627 struct kgsl_syncsource *syncsource)
628{
629 struct kgsl_syncsource_fence *sfence, *next;
630
631 spin_lock(&private->syncsource_lock);
632 if (syncsource->id != 0) {
633 idr_remove(&private->syncsource_idr, syncsource->id);
634 syncsource->id = 0;
635 }
636 spin_unlock(&private->syncsource_lock);
637
638 /* Signal all fences to release any callbacks */
639 spin_lock(&syncsource->lock);
640
641 list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
642 child_list) {
643 fence_signal_locked(&sfence->fence);
644 list_del_init(&sfence->child_list);
645 }
646
647 spin_unlock(&syncsource->lock);
648
649 /* put reference from syncsource creation */
650 kgsl_syncsource_put(syncsource);
651}
652
Shrenuj Bansala419c792016-10-20 14:05:11 -0700653long kgsl_ioctl_syncsource_destroy(struct kgsl_device_private *dev_priv,
654 unsigned int cmd, void *data)
655{
656 struct kgsl_syncsource_destroy *param = data;
657 struct kgsl_syncsource *syncsource = NULL;
658 struct kgsl_process_private *private = dev_priv->process_priv;
659
660 spin_lock(&private->syncsource_lock);
661 syncsource = idr_find(&private->syncsource_idr, param->id);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700662 spin_unlock(&private->syncsource_lock);
663
664 if (syncsource == NULL)
665 return -EINVAL;
666
Lynus Vazc031a9b2017-01-25 13:00:13 +0530667 kgsl_syncsource_cleanup(private, syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700668 return 0;
669}
670
671long kgsl_ioctl_syncsource_create_fence(struct kgsl_device_private *dev_priv,
672 unsigned int cmd, void *data)
673{
674 struct kgsl_syncsource_create_fence *param = data;
675 struct kgsl_syncsource *syncsource = NULL;
676 int ret = -EINVAL;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530677 struct kgsl_syncsource_fence *sfence = NULL;
678 struct sync_file *sync_file = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700679 int fd = -1;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700680
Lynus Vazc031a9b2017-01-25 13:00:13 +0530681 /*
682 * Take a refcount that is released when the fence is released
683 * (or if fence can't be added to the syncsource).
684 */
Shrenuj Bansala419c792016-10-20 14:05:11 -0700685 syncsource = kgsl_syncsource_get(dev_priv->process_priv,
686 param->id);
687 if (syncsource == NULL)
688 goto out;
689
Lynus Vazc031a9b2017-01-25 13:00:13 +0530690 sfence = kzalloc(sizeof(*sfence), GFP_KERNEL);
691 if (sfence == NULL) {
692 ret = -ENOMEM;
693 goto out;
694 }
695 sfence->parent = syncsource;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700696
Lynus Vazc031a9b2017-01-25 13:00:13 +0530697 /* Use a new fence context for each fence */
698 fence_init(&sfence->fence, &kgsl_syncsource_fence_ops,
699 &syncsource->lock, fence_context_alloc(1), 1);
700
701 sync_file = sync_file_create(&sfence->fence);
702
703 if (sync_file == NULL) {
704 KGSL_DRV_ERR(dev_priv->device, "Create sync_file failed\n");
Shrenuj Bansala419c792016-10-20 14:05:11 -0700705 ret = -ENOMEM;
706 goto out;
707 }
708
709 fd = get_unused_fd_flags(0);
710 if (fd < 0) {
711 ret = -EBADF;
712 goto out;
713 }
714 ret = 0;
715
Lynus Vazc031a9b2017-01-25 13:00:13 +0530716 fd_install(fd, sync_file->file);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700717
718 param->fence_fd = fd;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530719
720 spin_lock(&syncsource->lock);
721 list_add_tail(&sfence->child_list, &syncsource->child_list_head);
722 spin_unlock(&syncsource->lock);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700723out:
Lynus Vaz9d6334b2017-06-27 14:57:56 +0530724 /*
725 * We're transferring ownership of the fence to the sync file.
726 * The sync file takes an extra refcount when it is created, so put
727 * our refcount.
728 */
729 if (sync_file)
730 fence_put(&sfence->fence);
731
Shrenuj Bansala419c792016-10-20 14:05:11 -0700732 if (ret) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530733 if (sync_file)
734 fput(sync_file->file);
735 else if (sfence)
736 fence_put(&sfence->fence);
737 kgsl_syncsource_put(syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700738 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530739
740 return ret;
741}
742
743static int kgsl_syncsource_signal(struct kgsl_syncsource *syncsource,
744 struct fence *fence)
745{
746 struct kgsl_syncsource_fence *sfence, *next;
747 int ret = -EINVAL;
748
749 spin_lock(&syncsource->lock);
750
751 list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
752 child_list) {
753 if (fence == &sfence->fence) {
754 fence_signal_locked(fence);
755 list_del_init(&sfence->child_list);
756
757 ret = 0;
758 break;
759 }
760 }
761
762 spin_unlock(&syncsource->lock);
763
Shrenuj Bansala419c792016-10-20 14:05:11 -0700764 return ret;
765}
766
767long kgsl_ioctl_syncsource_signal_fence(struct kgsl_device_private *dev_priv,
768 unsigned int cmd, void *data)
769{
770 int ret = -EINVAL;
771 struct kgsl_syncsource_signal_fence *param = data;
772 struct kgsl_syncsource *syncsource = NULL;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530773 struct fence *fence = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700774
775 syncsource = kgsl_syncsource_get(dev_priv->process_priv,
776 param->id);
777 if (syncsource == NULL)
778 goto out;
779
Lynus Vazc031a9b2017-01-25 13:00:13 +0530780 fence = sync_file_get_fence(param->fence_fd);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700781 if (fence == NULL) {
782 ret = -EBADF;
783 goto out;
784 }
785
Lynus Vazc031a9b2017-01-25 13:00:13 +0530786 ret = kgsl_syncsource_signal(syncsource, fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700787out:
788 if (fence)
Lynus Vazc031a9b2017-01-25 13:00:13 +0530789 fence_put(fence);
790 if (syncsource)
791 kgsl_syncsource_put(syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700792 return ret;
793}
Lynus Vazc031a9b2017-01-25 13:00:13 +0530794
795static void kgsl_syncsource_fence_release(struct fence *fence)
796{
797 struct kgsl_syncsource_fence *sfence =
798 (struct kgsl_syncsource_fence *)fence;
799
800 /* Signal if it's not signaled yet */
801 kgsl_syncsource_signal(sfence->parent, fence);
802
803 /* Release the refcount on the syncsource */
804 kgsl_syncsource_put(sfence->parent);
805
806 kfree(sfence);
807}
808
809static const char *kgsl_syncsource_get_timeline_name(struct fence *fence)
810{
811 struct kgsl_syncsource_fence *sfence =
812 (struct kgsl_syncsource_fence *)fence;
813 struct kgsl_syncsource *syncsource = sfence->parent;
814
815 return syncsource->name;
816}
817
818static bool kgsl_syncsource_enable_signaling(struct fence *fence)
819{
820 return true;
821}
822
823static const char *kgsl_syncsource_driver_name(struct fence *fence)
824{
825 return "kgsl-syncsource-timeline";
826}
827
Lynus Vaz6aa3f5a2017-04-26 19:29:05 +0530828static void kgsl_syncsource_fence_value_str(struct fence *fence,
829 char *str, int size)
830{
831 /*
832 * Each fence is independent of the others on the same timeline.
833 * We use a different context for each of them.
834 */
835 snprintf(str, size, "%llu", fence->context);
836}
837
Lynus Vazc031a9b2017-01-25 13:00:13 +0530838static const struct fence_ops kgsl_syncsource_fence_ops = {
839 .get_driver_name = kgsl_syncsource_driver_name,
840 .get_timeline_name = kgsl_syncsource_get_timeline_name,
841 .enable_signaling = kgsl_syncsource_enable_signaling,
842 .wait = fence_default_wait,
843 .release = kgsl_syncsource_fence_release,
Lynus Vaz6aa3f5a2017-04-26 19:29:05 +0530844
845 .fence_value_str = kgsl_syncsource_fence_value_str,
Lynus Vazc031a9b2017-01-25 13:00:13 +0530846};
847