blob: 8f8e3e983ca6343bdb35d5025fd4d0d7064d7b52 [file] [log] [blame]
Shrenuj Bansala419c792016-10-20 14:05:11 -07001/* Copyright (c) 2012-2017, The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
13
14#include <linux/err.h>
15#include <linux/file.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18#include <linux/uaccess.h>
19
20#include <asm/current.h>
21
22#include "kgsl_sync.h"
23
Lynus Vazc031a9b2017-01-25 13:00:13 +053024static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *timeline,
Shrenuj Bansala419c792016-10-20 14:05:11 -070025 unsigned int timestamp);
26
Lynus Vazc031a9b2017-01-25 13:00:13 +053027static const struct fence_ops kgsl_sync_fence_ops;
28
29static struct kgsl_sync_fence *kgsl_sync_fence_create(
30 struct kgsl_context *context,
31 unsigned int timestamp)
Shrenuj Bansala419c792016-10-20 14:05:11 -070032{
Lynus Vazc031a9b2017-01-25 13:00:13 +053033 struct kgsl_sync_fence *kfence;
34 struct kgsl_sync_timeline *ktimeline = context->ktimeline;
35 unsigned long flags;
Shrenuj Bansala419c792016-10-20 14:05:11 -070036
Lynus Vazc031a9b2017-01-25 13:00:13 +053037 /* Get a refcount to the timeline. Put when released */
38 if (!kref_get_unless_zero(&ktimeline->kref))
39 return NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -070040
Lynus Vazc031a9b2017-01-25 13:00:13 +053041 kfence = kzalloc(sizeof(*kfence), GFP_KERNEL);
42 if (kfence == NULL) {
43 kgsl_sync_timeline_put(ktimeline);
44 KGSL_DRV_ERR(context->device, "Couldn't allocate fence\n");
45 return NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -070046 }
Lynus Vazc031a9b2017-01-25 13:00:13 +053047
48 kfence->parent = ktimeline;
49 kfence->context_id = context->id;
50 kfence->timestamp = timestamp;
51
52 fence_init(&kfence->fence, &kgsl_sync_fence_ops, &ktimeline->lock,
53 ktimeline->fence_context, timestamp);
54
Lynus Vaz9d6334b2017-06-27 14:57:56 +053055 /*
56 * sync_file_create() takes a refcount to the fence. This refcount is
57 * put when the fence is signaled.
58 */
Lynus Vazc031a9b2017-01-25 13:00:13 +053059 kfence->sync_file = sync_file_create(&kfence->fence);
60
61 if (kfence->sync_file == NULL) {
62 kgsl_sync_timeline_put(ktimeline);
63 KGSL_DRV_ERR(context->device, "Create sync_file failed\n");
64 kfree(kfence);
65 return NULL;
66 }
67
Lynus Vazc031a9b2017-01-25 13:00:13 +053068 spin_lock_irqsave(&ktimeline->lock, flags);
69 list_add_tail(&kfence->child_list, &ktimeline->child_list_head);
70 spin_unlock_irqrestore(&ktimeline->lock, flags);
71
72 return kfence;
Shrenuj Bansala419c792016-10-20 14:05:11 -070073}
74
Lynus Vazc031a9b2017-01-25 13:00:13 +053075static void kgsl_sync_fence_release(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070076{
Lynus Vazc031a9b2017-01-25 13:00:13 +053077 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
78
79 kgsl_sync_timeline_put(kfence->parent);
80 kfree(kfence);
Shrenuj Bansala419c792016-10-20 14:05:11 -070081}
82
Lynus Vazc031a9b2017-01-25 13:00:13 +053083/* Called with ktimeline->lock held */
84bool kgsl_sync_fence_has_signaled(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070085{
Lynus Vazc031a9b2017-01-25 13:00:13 +053086 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
87 struct kgsl_sync_timeline *ktimeline = kfence->parent;
88 unsigned int ts = kfence->timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -070089
Lynus Vazc031a9b2017-01-25 13:00:13 +053090 return (timestamp_cmp(ktimeline->last_timestamp, ts) >= 0);
Shrenuj Bansala419c792016-10-20 14:05:11 -070091}
92
Lynus Vazc031a9b2017-01-25 13:00:13 +053093bool kgsl_enable_signaling(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -070094{
Lynus Vazc031a9b2017-01-25 13:00:13 +053095 return !kgsl_sync_fence_has_signaled(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -070096}
97
Lynus Vazc031a9b2017-01-25 13:00:13 +053098struct kgsl_sync_fence_event_priv {
Shrenuj Bansala419c792016-10-20 14:05:11 -070099 struct kgsl_context *context;
100 unsigned int timestamp;
101};
102
103/**
Lynus Vazc031a9b2017-01-25 13:00:13 +0530104 * kgsl_sync_fence_event_cb - Event callback for a fence timestamp event
Shrenuj Bansala419c792016-10-20 14:05:11 -0700105 * @device - The KGSL device that expired the timestamp
106 * @context- Pointer to the context that owns the event
107 * @priv: Private data for the callback
108 * @result - Result of the event (retired or canceled)
109 *
110 * Signal a fence following the expiration of a timestamp
111 */
112
Lynus Vazc031a9b2017-01-25 13:00:13 +0530113static void kgsl_sync_fence_event_cb(struct kgsl_device *device,
Shrenuj Bansala419c792016-10-20 14:05:11 -0700114 struct kgsl_event_group *group, void *priv, int result)
115{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530116 struct kgsl_sync_fence_event_priv *ev = priv;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700117
Lynus Vazc031a9b2017-01-25 13:00:13 +0530118 kgsl_sync_timeline_signal(ev->context->ktimeline, ev->timestamp);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700119 kgsl_context_put(ev->context);
120 kfree(ev);
121}
122
123static int _add_fence_event(struct kgsl_device *device,
124 struct kgsl_context *context, unsigned int timestamp)
125{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530126 struct kgsl_sync_fence_event_priv *event;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700127 int ret;
128
129 event = kmalloc(sizeof(*event), GFP_KERNEL);
130 if (event == NULL)
131 return -ENOMEM;
132
133 /*
134 * Increase the refcount for the context to keep it through the
135 * callback
136 */
137 if (!_kgsl_context_get(context)) {
138 kfree(event);
139 return -ENOENT;
140 }
141
142 event->context = context;
143 event->timestamp = timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700144
145 ret = kgsl_add_event(device, &context->events, timestamp,
Lynus Vazc031a9b2017-01-25 13:00:13 +0530146 kgsl_sync_fence_event_cb, event);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700147
148 if (ret) {
149 kgsl_context_put(context);
150 kfree(event);
151 }
152
153 return ret;
154}
155
Lynus Vazc50d6202017-03-23 13:42:58 +0530156/* Only to be used if creating a related event failed */
157static void kgsl_sync_cancel(struct kgsl_sync_fence *kfence)
158{
159 spin_lock(&kfence->parent->lock);
160 if (!list_empty(&kfence->child_list)) {
161 list_del_init(&kfence->child_list);
162 fence_put(&kfence->fence);
163 }
164 spin_unlock(&kfence->parent->lock);
165}
166
Shrenuj Bansala419c792016-10-20 14:05:11 -0700167/**
168 * kgsl_add_fence_event - Create a new fence event
169 * @device - KGSL device to create the event on
170 * @timestamp - Timestamp to trigger the event
171 * @data - Return fence fd stored in struct kgsl_timestamp_event_fence
172 * @len - length of the fence event
173 * @owner - driver instance that owns this event
174 * @returns 0 on success or error code on error
175 *
176 * Create a fence and register an event to signal the fence when
177 * the timestamp expires
178 */
179
180int kgsl_add_fence_event(struct kgsl_device *device,
181 u32 context_id, u32 timestamp, void __user *data, int len,
182 struct kgsl_device_private *owner)
183{
184 struct kgsl_timestamp_event_fence priv;
185 struct kgsl_context *context;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530186 struct kgsl_sync_fence *kfence = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700187 int ret = -EINVAL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700188 unsigned int cur;
189
190 priv.fence_fd = -1;
191
192 if (len != sizeof(priv))
193 return -EINVAL;
194
195 context = kgsl_context_get_owner(owner, context_id);
196
197 if (context == NULL)
198 return -EINVAL;
199
200 if (test_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv))
201 goto out;
202
Lynus Vazc031a9b2017-01-25 13:00:13 +0530203 kfence = kgsl_sync_fence_create(context, timestamp);
204 if (kfence == NULL) {
205 KGSL_DRV_CRIT_RATELIMIT(device,
206 "kgsl_sync_fence_create failed\n");
Shrenuj Bansala419c792016-10-20 14:05:11 -0700207 ret = -ENOMEM;
208 goto out;
209 }
210
211 priv.fence_fd = get_unused_fd_flags(0);
212 if (priv.fence_fd < 0) {
213 KGSL_DRV_CRIT_RATELIMIT(device,
214 "Unable to get a file descriptor: %d\n",
215 priv.fence_fd);
216 ret = priv.fence_fd;
217 goto out;
218 }
219
220 /*
221 * If the timestamp hasn't expired yet create an event to trigger it.
222 * Otherwise, just signal the fence - there is no reason to go through
223 * the effort of creating a fence we don't need.
224 */
225
226 kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &cur);
227
228 if (timestamp_cmp(cur, timestamp) >= 0) {
229 ret = 0;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530230 kgsl_sync_timeline_signal(context->ktimeline, cur);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700231 } else {
232 ret = _add_fence_event(device, context, timestamp);
233 if (ret)
234 goto out;
235 }
236
237 if (copy_to_user(data, &priv, sizeof(priv))) {
238 ret = -EFAULT;
239 goto out;
240 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530241 fd_install(priv.fence_fd, kfence->sync_file->file);
242
Shrenuj Bansala419c792016-10-20 14:05:11 -0700243out:
244 kgsl_context_put(context);
245 if (ret) {
246 if (priv.fence_fd >= 0)
247 put_unused_fd(priv.fence_fd);
248
Lynus Vazc031a9b2017-01-25 13:00:13 +0530249 if (kfence) {
Lynus Vazc50d6202017-03-23 13:42:58 +0530250 kgsl_sync_cancel(kfence);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530251 /*
252 * Put the refcount of sync file. This will release
253 * kfence->fence as well.
254 */
255 fput(kfence->sync_file->file);
256 }
Shrenuj Bansala419c792016-10-20 14:05:11 -0700257 }
258 return ret;
259}
260
Lynus Vazc031a9b2017-01-25 13:00:13 +0530261static unsigned int kgsl_sync_fence_get_timestamp(
262 struct kgsl_sync_timeline *ktimeline,
263 enum kgsl_timestamp_type type)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700264{
265 unsigned int ret = 0;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700266
267 if (ktimeline->device == NULL)
268 return 0;
269
Lynus Vazc031a9b2017-01-25 13:00:13 +0530270 kgsl_readtimestamp(ktimeline->device, ktimeline->context, type, &ret);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700271
Shrenuj Bansala419c792016-10-20 14:05:11 -0700272 return ret;
273}
274
Lynus Vazc031a9b2017-01-25 13:00:13 +0530275static void kgsl_sync_timeline_value_str(struct fence *fence,
276 char *str, int size)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700277{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530278 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
279 struct kgsl_sync_timeline *ktimeline = kfence->parent;
280
281 unsigned int timestamp_retired;
282 unsigned int timestamp_queued;
283
284 if (!kref_get_unless_zero(&ktimeline->kref))
285 return;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700286
287 /*
288 * This callback can be called before the device and spinlock are
289 * initialized in struct kgsl_sync_timeline. kgsl_sync_get_timestamp()
290 * will check if device is NULL and return 0. Queued and retired
291 * timestamp of the context will be reported as 0, which is correct
292 * because the context and timeline are just getting initialized.
293 */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530294 timestamp_retired = kgsl_sync_fence_get_timestamp(ktimeline,
295 KGSL_TIMESTAMP_RETIRED);
296 timestamp_queued = kgsl_sync_fence_get_timestamp(ktimeline,
297 KGSL_TIMESTAMP_QUEUED);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700298
299 snprintf(str, size, "%u queued:%u retired:%u",
300 ktimeline->last_timestamp,
301 timestamp_queued, timestamp_retired);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530302
303 kgsl_sync_timeline_put(ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700304}
305
Lynus Vazc031a9b2017-01-25 13:00:13 +0530306static void kgsl_sync_fence_value_str(struct fence *fence, char *str, int size)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700307{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530308 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700309
Lynus Vazc031a9b2017-01-25 13:00:13 +0530310 snprintf(str, size, "%u", kfence->timestamp);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700311}
312
Lynus Vazc031a9b2017-01-25 13:00:13 +0530313static const char *kgsl_sync_fence_driver_name(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700314{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530315 return "kgsl-timeline";
Shrenuj Bansala419c792016-10-20 14:05:11 -0700316}
317
Lynus Vazc031a9b2017-01-25 13:00:13 +0530318static const char *kgsl_sync_timeline_name(struct fence *fence)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700319{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530320 struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
321 struct kgsl_sync_timeline *ktimeline = kfence->parent;
322
323 return ktimeline->name;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700324}
Shrenuj Bansala419c792016-10-20 14:05:11 -0700325
326int kgsl_sync_timeline_create(struct kgsl_context *context)
327{
328 struct kgsl_sync_timeline *ktimeline;
329
330 /*
331 * Generate a name which includes the thread name, thread id, process
332 * name, process id, and context id. This makes it possible to
333 * identify the context of a timeline in the sync dump.
334 */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530335 char ktimeline_name[sizeof(ktimeline->name)] = {};
336
337 /* Put context when timeline is released */
338 if (!_kgsl_context_get(context))
339 return -ENOENT;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700340
341 snprintf(ktimeline_name, sizeof(ktimeline_name),
342 "%s_%.15s(%d)-%.15s(%d)-%d",
343 context->device->name,
344 current->group_leader->comm, current->group_leader->pid,
345 current->comm, current->pid, context->id);
346
Lynus Vazc031a9b2017-01-25 13:00:13 +0530347 ktimeline = kzalloc(sizeof(*ktimeline), GFP_KERNEL);
348 if (ktimeline == NULL) {
349 kgsl_context_put(context);
350 return -ENOMEM;
351 }
Shrenuj Bansala419c792016-10-20 14:05:11 -0700352
Lynus Vazc031a9b2017-01-25 13:00:13 +0530353 kref_init(&ktimeline->kref);
354 strlcpy(ktimeline->name, ktimeline_name, KGSL_TIMELINE_NAME_LEN);
355 ktimeline->fence_context = fence_context_alloc(1);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700356 ktimeline->last_timestamp = 0;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530357 INIT_LIST_HEAD(&ktimeline->child_list_head);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700358 spin_lock_init(&ktimeline->lock);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530359 ktimeline->device = context->device;
360 ktimeline->context = context;
361
362 context->ktimeline = ktimeline;
363
Shrenuj Bansala419c792016-10-20 14:05:11 -0700364 return 0;
365}
366
Lynus Vazc031a9b2017-01-25 13:00:13 +0530367static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *ktimeline,
368 unsigned int timestamp)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700369{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530370 unsigned long flags;
371 struct kgsl_sync_fence *kfence, *next;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700372
Lynus Vazc031a9b2017-01-25 13:00:13 +0530373 kref_get(&ktimeline->kref);
374
375 spin_lock_irqsave(&ktimeline->lock, flags);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700376 if (timestamp_cmp(timestamp, ktimeline->last_timestamp) > 0)
377 ktimeline->last_timestamp = timestamp;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700378
Lynus Vazc031a9b2017-01-25 13:00:13 +0530379 list_for_each_entry_safe(kfence, next, &ktimeline->child_list_head,
380 child_list) {
381 if (fence_is_signaled_locked(&kfence->fence)) {
Lynus Vazc50d6202017-03-23 13:42:58 +0530382 list_del_init(&kfence->child_list);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530383 fence_put(&kfence->fence);
384 }
385 }
386
387 spin_unlock_irqrestore(&ktimeline->lock, flags);
388 kgsl_sync_timeline_put(ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700389}
390
391void kgsl_sync_timeline_destroy(struct kgsl_context *context)
392{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530393 kfree(context->ktimeline);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700394}
395
Lynus Vazc031a9b2017-01-25 13:00:13 +0530396static void kgsl_sync_timeline_release(struct kref *kref)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700397{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530398 struct kgsl_sync_timeline *ktimeline =
399 container_of(kref, struct kgsl_sync_timeline, kref);
400
401 /*
402 * Only put the context refcount here. The context destroy function
403 * will call kgsl_sync_timeline_destroy() to kfree it
404 */
405 kgsl_context_put(ktimeline->context);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700406}
407
Lynus Vazc031a9b2017-01-25 13:00:13 +0530408void kgsl_sync_timeline_put(struct kgsl_sync_timeline *ktimeline)
409{
410 if (ktimeline)
411 kref_put(&ktimeline->kref, kgsl_sync_timeline_release);
412}
413
414static const struct fence_ops kgsl_sync_fence_ops = {
415 .get_driver_name = kgsl_sync_fence_driver_name,
416 .get_timeline_name = kgsl_sync_timeline_name,
417 .enable_signaling = kgsl_enable_signaling,
418 .signaled = kgsl_sync_fence_has_signaled,
419 .wait = fence_default_wait,
420 .release = kgsl_sync_fence_release,
421
422 .fence_value_str = kgsl_sync_fence_value_str,
423 .timeline_value_str = kgsl_sync_timeline_value_str,
424};
425
426static void kgsl_sync_fence_callback(struct fence *fence, struct fence_cb *cb)
427{
428 struct kgsl_sync_fence_cb *kcb = (struct kgsl_sync_fence_cb *)cb;
429
Lynus Vaz27da44d2017-07-26 13:50:10 +0530430 /*
431 * If the callback is marked for cancellation in a separate thread,
432 * let the other thread do the cleanup.
433 */
434 if (kcb->func(kcb->priv)) {
435 fence_put(kcb->fence);
436 kfree(kcb);
437 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530438}
439
Lynus Vaze99b92b2017-04-24 18:04:54 +0530440static void kgsl_get_fence_name(struct fence *fence,
441 char *fence_name, int name_len)
442{
443 char *ptr = fence_name;
444 char *last = fence_name + name_len;
445
446 ptr += snprintf(ptr, last - ptr, "%s %s",
447 fence->ops->get_driver_name(fence),
448 fence->ops->get_timeline_name(fence));
449
450 if ((ptr + 2) >= last)
451 return;
452
453 if (fence->ops->fence_value_str) {
454 ptr += snprintf(ptr, last - ptr, ": ");
455 fence->ops->fence_value_str(fence, ptr, last - ptr);
456 }
457}
458
Lynus Vazc031a9b2017-01-25 13:00:13 +0530459struct kgsl_sync_fence_cb *kgsl_sync_fence_async_wait(int fd,
Lynus Vaz27da44d2017-07-26 13:50:10 +0530460 bool (*func)(void *priv), void *priv, char *fence_name, int name_len)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700461{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530462 struct kgsl_sync_fence_cb *kcb;
463 struct fence *fence;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700464 int status;
465
Lynus Vazc031a9b2017-01-25 13:00:13 +0530466 fence = sync_file_get_fence(fd);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700467 if (fence == NULL)
468 return ERR_PTR(-EINVAL);
469
Lynus Vazc031a9b2017-01-25 13:00:13 +0530470 /* create the callback */
471 kcb = kzalloc(sizeof(*kcb), GFP_ATOMIC);
472 if (kcb == NULL) {
473 fence_put(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700474 return ERR_PTR(-ENOMEM);
475 }
476
Lynus Vazc031a9b2017-01-25 13:00:13 +0530477 kcb->fence = fence;
478 kcb->priv = priv;
479 kcb->func = func;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700480
Lynus Vaze99b92b2017-04-24 18:04:54 +0530481 if (fence_name)
482 kgsl_get_fence_name(fence, fence_name, name_len);
483
Shrenuj Bansala419c792016-10-20 14:05:11 -0700484 /* if status then error or signaled */
Lynus Vazc031a9b2017-01-25 13:00:13 +0530485 status = fence_add_callback(fence, &kcb->fence_cb,
486 kgsl_sync_fence_callback);
487
Shrenuj Bansala419c792016-10-20 14:05:11 -0700488 if (status) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530489 kfree(kcb);
Harshdeep Dhatt0ed60a42017-04-20 16:41:50 -0600490 if (!fence_is_signaled(fence))
Lynus Vazc031a9b2017-01-25 13:00:13 +0530491 kcb = ERR_PTR(status);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700492 else
Lynus Vazc031a9b2017-01-25 13:00:13 +0530493 kcb = NULL;
494 fence_put(fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700495 }
496
Lynus Vazc031a9b2017-01-25 13:00:13 +0530497 return kcb;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700498}
499
Lynus Vaz27da44d2017-07-26 13:50:10 +0530500/*
501 * Cancel the fence async callback and do the cleanup. The caller must make
502 * sure that the callback (if run before cancelling) returns false, so that
503 * no other thread frees the pointer.
504 */
505void kgsl_sync_fence_async_cancel(struct kgsl_sync_fence_cb *kcb)
Shrenuj Bansala419c792016-10-20 14:05:11 -0700506{
Lynus Vazc031a9b2017-01-25 13:00:13 +0530507 if (kcb == NULL)
Lynus Vaz27da44d2017-07-26 13:50:10 +0530508 return;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700509
Lynus Vaz27da44d2017-07-26 13:50:10 +0530510 /*
511 * After fence_remove_callback() returns, the fence callback is
512 * either not called at all, or completed without freeing kcb.
513 * This thread can then put the fence refcount and free kcb.
514 */
515 fence_remove_callback(kcb->fence, &kcb->fence_cb);
516 fence_put(kcb->fence);
517 kfree(kcb);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700518}
519
Shrenuj Bansala419c792016-10-20 14:05:11 -0700520struct kgsl_syncsource {
521 struct kref refcount;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530522 char name[KGSL_TIMELINE_NAME_LEN];
Shrenuj Bansala419c792016-10-20 14:05:11 -0700523 int id;
524 struct kgsl_process_private *private;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530525 struct list_head child_list_head;
526 spinlock_t lock;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700527};
528
Lynus Vazc031a9b2017-01-25 13:00:13 +0530529struct kgsl_syncsource_fence {
530 struct fence fence;
531 struct kgsl_syncsource *parent;
532 struct list_head child_list;
533};
534
535static const struct fence_ops kgsl_syncsource_fence_ops;
536
Shrenuj Bansala419c792016-10-20 14:05:11 -0700537long kgsl_ioctl_syncsource_create(struct kgsl_device_private *dev_priv,
538 unsigned int cmd, void *data)
539{
540 struct kgsl_syncsource *syncsource = NULL;
541 struct kgsl_syncsource_create *param = data;
542 int ret = -EINVAL;
543 int id = 0;
544 struct kgsl_process_private *private = dev_priv->process_priv;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530545 char name[KGSL_TIMELINE_NAME_LEN];
546
547 if (!kgsl_process_private_get(private))
548 return ret;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700549
550 syncsource = kzalloc(sizeof(*syncsource), GFP_KERNEL);
551 if (syncsource == NULL) {
552 ret = -ENOMEM;
553 goto out;
554 }
555
556 snprintf(name, sizeof(name), "kgsl-syncsource-pid-%d",
557 current->group_leader->pid);
558
Shrenuj Bansala419c792016-10-20 14:05:11 -0700559 kref_init(&syncsource->refcount);
Lynus Vazc031a9b2017-01-25 13:00:13 +0530560 strlcpy(syncsource->name, name, KGSL_TIMELINE_NAME_LEN);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700561 syncsource->private = private;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530562 INIT_LIST_HEAD(&syncsource->child_list_head);
563 spin_lock_init(&syncsource->lock);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700564
565 idr_preload(GFP_KERNEL);
566 spin_lock(&private->syncsource_lock);
567 id = idr_alloc(&private->syncsource_idr, syncsource, 1, 0, GFP_NOWAIT);
568 if (id > 0) {
569 syncsource->id = id;
570 param->id = id;
571 ret = 0;
572 } else {
573 ret = id;
574 }
575
576 spin_unlock(&private->syncsource_lock);
577 idr_preload_end();
578
579out:
580 if (ret) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530581 kgsl_process_private_put(private);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700582 kfree(syncsource);
583 }
584
585 return ret;
586}
587
588static struct kgsl_syncsource *
589kgsl_syncsource_get(struct kgsl_process_private *private, int id)
590{
591 int result = 0;
592 struct kgsl_syncsource *syncsource = NULL;
593
594 spin_lock(&private->syncsource_lock);
595
596 syncsource = idr_find(&private->syncsource_idr, id);
597 if (syncsource)
598 result = kref_get_unless_zero(&syncsource->refcount);
599
600 spin_unlock(&private->syncsource_lock);
601
602 return result ? syncsource : NULL;
603}
604
605static void kgsl_syncsource_destroy(struct kref *kref)
606{
607 struct kgsl_syncsource *syncsource = container_of(kref,
608 struct kgsl_syncsource,
609 refcount);
610
611 struct kgsl_process_private *private = syncsource->private;
612
Lynus Vazc031a9b2017-01-25 13:00:13 +0530613 /* Done with process private. Release the refcount */
614 kgsl_process_private_put(private);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700615
616 kfree(syncsource);
617}
618
619void kgsl_syncsource_put(struct kgsl_syncsource *syncsource)
620{
621 if (syncsource)
622 kref_put(&syncsource->refcount, kgsl_syncsource_destroy);
623}
624
Lynus Vazc031a9b2017-01-25 13:00:13 +0530625void kgsl_syncsource_cleanup(struct kgsl_process_private *private,
626 struct kgsl_syncsource *syncsource)
627{
628 struct kgsl_syncsource_fence *sfence, *next;
629
630 spin_lock(&private->syncsource_lock);
631 if (syncsource->id != 0) {
632 idr_remove(&private->syncsource_idr, syncsource->id);
633 syncsource->id = 0;
634 }
635 spin_unlock(&private->syncsource_lock);
636
637 /* Signal all fences to release any callbacks */
638 spin_lock(&syncsource->lock);
639
640 list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
641 child_list) {
642 fence_signal_locked(&sfence->fence);
643 list_del_init(&sfence->child_list);
644 }
645
646 spin_unlock(&syncsource->lock);
647
648 /* put reference from syncsource creation */
649 kgsl_syncsource_put(syncsource);
650}
651
Shrenuj Bansala419c792016-10-20 14:05:11 -0700652long kgsl_ioctl_syncsource_destroy(struct kgsl_device_private *dev_priv,
653 unsigned int cmd, void *data)
654{
655 struct kgsl_syncsource_destroy *param = data;
656 struct kgsl_syncsource *syncsource = NULL;
657 struct kgsl_process_private *private = dev_priv->process_priv;
658
659 spin_lock(&private->syncsource_lock);
660 syncsource = idr_find(&private->syncsource_idr, param->id);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700661 spin_unlock(&private->syncsource_lock);
662
663 if (syncsource == NULL)
664 return -EINVAL;
665
Lynus Vazc031a9b2017-01-25 13:00:13 +0530666 kgsl_syncsource_cleanup(private, syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700667 return 0;
668}
669
670long kgsl_ioctl_syncsource_create_fence(struct kgsl_device_private *dev_priv,
671 unsigned int cmd, void *data)
672{
673 struct kgsl_syncsource_create_fence *param = data;
674 struct kgsl_syncsource *syncsource = NULL;
675 int ret = -EINVAL;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530676 struct kgsl_syncsource_fence *sfence = NULL;
677 struct sync_file *sync_file = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700678 int fd = -1;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700679
Lynus Vazc031a9b2017-01-25 13:00:13 +0530680 /*
681 * Take a refcount that is released when the fence is released
682 * (or if fence can't be added to the syncsource).
683 */
Shrenuj Bansala419c792016-10-20 14:05:11 -0700684 syncsource = kgsl_syncsource_get(dev_priv->process_priv,
685 param->id);
686 if (syncsource == NULL)
687 goto out;
688
Lynus Vazc031a9b2017-01-25 13:00:13 +0530689 sfence = kzalloc(sizeof(*sfence), GFP_KERNEL);
690 if (sfence == NULL) {
691 ret = -ENOMEM;
692 goto out;
693 }
694 sfence->parent = syncsource;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700695
Lynus Vazc031a9b2017-01-25 13:00:13 +0530696 /* Use a new fence context for each fence */
697 fence_init(&sfence->fence, &kgsl_syncsource_fence_ops,
698 &syncsource->lock, fence_context_alloc(1), 1);
699
700 sync_file = sync_file_create(&sfence->fence);
701
702 if (sync_file == NULL) {
703 KGSL_DRV_ERR(dev_priv->device, "Create sync_file failed\n");
Shrenuj Bansala419c792016-10-20 14:05:11 -0700704 ret = -ENOMEM;
705 goto out;
706 }
707
708 fd = get_unused_fd_flags(0);
709 if (fd < 0) {
710 ret = -EBADF;
711 goto out;
712 }
713 ret = 0;
714
Lynus Vazc031a9b2017-01-25 13:00:13 +0530715 fd_install(fd, sync_file->file);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700716
717 param->fence_fd = fd;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530718
719 spin_lock(&syncsource->lock);
720 list_add_tail(&sfence->child_list, &syncsource->child_list_head);
721 spin_unlock(&syncsource->lock);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700722out:
Lynus Vaz9d6334b2017-06-27 14:57:56 +0530723 /*
724 * We're transferring ownership of the fence to the sync file.
725 * The sync file takes an extra refcount when it is created, so put
726 * our refcount.
727 */
728 if (sync_file)
729 fence_put(&sfence->fence);
730
Shrenuj Bansala419c792016-10-20 14:05:11 -0700731 if (ret) {
Lynus Vazc031a9b2017-01-25 13:00:13 +0530732 if (sync_file)
733 fput(sync_file->file);
734 else if (sfence)
735 fence_put(&sfence->fence);
736 kgsl_syncsource_put(syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700737 }
Lynus Vazc031a9b2017-01-25 13:00:13 +0530738
739 return ret;
740}
741
742static int kgsl_syncsource_signal(struct kgsl_syncsource *syncsource,
743 struct fence *fence)
744{
745 struct kgsl_syncsource_fence *sfence, *next;
746 int ret = -EINVAL;
747
748 spin_lock(&syncsource->lock);
749
750 list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
751 child_list) {
752 if (fence == &sfence->fence) {
753 fence_signal_locked(fence);
754 list_del_init(&sfence->child_list);
755
756 ret = 0;
757 break;
758 }
759 }
760
761 spin_unlock(&syncsource->lock);
762
Shrenuj Bansala419c792016-10-20 14:05:11 -0700763 return ret;
764}
765
766long kgsl_ioctl_syncsource_signal_fence(struct kgsl_device_private *dev_priv,
767 unsigned int cmd, void *data)
768{
769 int ret = -EINVAL;
770 struct kgsl_syncsource_signal_fence *param = data;
771 struct kgsl_syncsource *syncsource = NULL;
Lynus Vazc031a9b2017-01-25 13:00:13 +0530772 struct fence *fence = NULL;
Shrenuj Bansala419c792016-10-20 14:05:11 -0700773
774 syncsource = kgsl_syncsource_get(dev_priv->process_priv,
775 param->id);
776 if (syncsource == NULL)
777 goto out;
778
Lynus Vazc031a9b2017-01-25 13:00:13 +0530779 fence = sync_file_get_fence(param->fence_fd);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700780 if (fence == NULL) {
781 ret = -EBADF;
782 goto out;
783 }
784
Lynus Vazc031a9b2017-01-25 13:00:13 +0530785 ret = kgsl_syncsource_signal(syncsource, fence);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700786out:
787 if (fence)
Lynus Vazc031a9b2017-01-25 13:00:13 +0530788 fence_put(fence);
789 if (syncsource)
790 kgsl_syncsource_put(syncsource);
Shrenuj Bansala419c792016-10-20 14:05:11 -0700791 return ret;
792}
Lynus Vazc031a9b2017-01-25 13:00:13 +0530793
794static void kgsl_syncsource_fence_release(struct fence *fence)
795{
796 struct kgsl_syncsource_fence *sfence =
797 (struct kgsl_syncsource_fence *)fence;
798
799 /* Signal if it's not signaled yet */
800 kgsl_syncsource_signal(sfence->parent, fence);
801
802 /* Release the refcount on the syncsource */
803 kgsl_syncsource_put(sfence->parent);
804
805 kfree(sfence);
806}
807
808static const char *kgsl_syncsource_get_timeline_name(struct fence *fence)
809{
810 struct kgsl_syncsource_fence *sfence =
811 (struct kgsl_syncsource_fence *)fence;
812 struct kgsl_syncsource *syncsource = sfence->parent;
813
814 return syncsource->name;
815}
816
817static bool kgsl_syncsource_enable_signaling(struct fence *fence)
818{
819 return true;
820}
821
822static const char *kgsl_syncsource_driver_name(struct fence *fence)
823{
824 return "kgsl-syncsource-timeline";
825}
826
Lynus Vaz6aa3f5a2017-04-26 19:29:05 +0530827static void kgsl_syncsource_fence_value_str(struct fence *fence,
828 char *str, int size)
829{
830 /*
831 * Each fence is independent of the others on the same timeline.
832 * We use a different context for each of them.
833 */
834 snprintf(str, size, "%llu", fence->context);
835}
836
Lynus Vazc031a9b2017-01-25 13:00:13 +0530837static const struct fence_ops kgsl_syncsource_fence_ops = {
838 .get_driver_name = kgsl_syncsource_driver_name,
839 .get_timeline_name = kgsl_syncsource_get_timeline_name,
840 .enable_signaling = kgsl_syncsource_enable_signaling,
841 .wait = fence_default_wait,
842 .release = kgsl_syncsource_fence_release,
Lynus Vaz6aa3f5a2017-04-26 19:29:05 +0530843
844 .fence_value_str = kgsl_syncsource_fence_value_str,
Lynus Vazc031a9b2017-01-25 13:00:13 +0530845};
846