blob: 9dc86d3303aedf5ecf378fa6daed87831177546c [file] [log] [blame]
Gustavo Padovan1867a232016-05-31 16:59:05 -03001/*
Gustavo Padovane912c882016-08-11 12:26:42 -03002 * Sync File validation framework
Gustavo Padovan1867a232016-05-31 16:59:05 -03003 *
4 * Copyright (C) 2012 Google, Inc.
5 *
6 * This software is licensed under the terms of the GNU General Public
7 * License version 2, as published by the Free Software Foundation, and
8 * may be copied, distributed, and modified under those terms.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 */
16
17#include <linux/file.h>
18#include <linux/fs.h>
19#include <linux/uaccess.h>
Gustavo Padovanaff9da12016-05-31 16:59:07 -030020#include <linux/slab.h>
Gustavo Padovan1867a232016-05-31 16:59:05 -030021#include <linux/sync_file.h>
22
Gustavo Padovan1fe82e22016-05-31 16:59:12 -030023#include "sync_debug.h"
Gustavo Padovan1867a232016-05-31 16:59:05 -030024
Gustavo Padovanaff9da12016-05-31 16:59:07 -030025#define CREATE_TRACE_POINTS
Gustavo Padovana04f9152016-08-11 12:26:41 -030026#include "sync_trace.h"
Gustavo Padovanaff9da12016-05-31 16:59:07 -030027
Gustavo Padovanfc0c9a02016-08-11 13:45:53 -030028/*
29 * SW SYNC validation framework
30 *
31 * A sync object driver that uses a 32bit counter to coordinate
32 * synchronization. Useful when there is no hardware primitive backing
33 * the synchronization.
34 *
35 * To start the framework just open:
36 *
37 * <debugfs>/sync/sw_sync
38 *
39 * That will create a sync timeline, all fences created under this timeline
40 * file descriptor will belong to the this timeline.
41 *
42 * The 'sw_sync' file can be opened many times as to create different
43 * timelines.
44 *
45 * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct
46 * sw_sync_ioctl_create_fence as parameter.
47 *
48 * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used
49 * with the increment as u32. This will update the last signaled value
50 * from the timeline and signal any fence that has a seqno smaller or equal
51 * to it.
52 *
53 * struct sw_sync_ioctl_create_fence
54 * @value: the seqno to initialise the fence with
55 * @name: the name of the new sync point
56 * @fence: return the fd of the new sync_file with the created fence
57 */
Gustavo Padovan6f65aa82016-05-31 16:59:08 -030058struct sw_sync_create_fence_data {
59 __u32 value;
60 char name[32];
61 __s32 fence; /* fd of new fence */
62};
63
64#define SW_SYNC_IOC_MAGIC 'W'
65
66#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\
67 struct sw_sync_create_fence_data)
Gustavo Padovanfc0c9a02016-08-11 13:45:53 -030068
Gustavo Padovan6f65aa82016-05-31 16:59:08 -030069#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
70
Gustavo Padovanaff9da12016-05-31 16:59:07 -030071static const struct fence_ops timeline_fence_ops;
72
73static inline struct sync_pt *fence_to_sync_pt(struct fence *fence)
74{
75 if (fence->ops != &timeline_fence_ops)
76 return NULL;
77 return container_of(fence, struct sync_pt, base);
78}
79
80/**
81 * sync_timeline_create() - creates a sync object
Gustavo Padovanaff9da12016-05-31 16:59:07 -030082 * @name: sync_timeline name
83 *
84 * Creates a new sync_timeline. Returns the sync_timeline object or NULL in
85 * case of error.
86 */
Gustavo Padovanb9bc2b72016-05-31 16:59:11 -030087struct sync_timeline *sync_timeline_create(const char *name)
Gustavo Padovanaff9da12016-05-31 16:59:07 -030088{
89 struct sync_timeline *obj;
90
91 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
92 if (!obj)
93 return NULL;
94
95 kref_init(&obj->kref);
96 obj->context = fence_context_alloc(1);
97 strlcpy(obj->name, name, sizeof(obj->name));
Gustavo Padovanaff9da12016-05-31 16:59:07 -030098
Chris Wilsondb767402017-06-29 22:12:53 +010099 obj->pt_tree = RB_ROOT;
Chris Wilsone82ecb22017-06-29 22:05:32 +0100100 INIT_LIST_HEAD(&obj->pt_list);
101 spin_lock_init(&obj->lock);
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300102
103 sync_timeline_debug_add(obj);
104
105 return obj;
106}
107
108static void sync_timeline_free(struct kref *kref)
109{
110 struct sync_timeline *obj =
111 container_of(kref, struct sync_timeline, kref);
112
113 sync_timeline_debug_remove(obj);
114
115 kfree(obj);
116}
117
118static void sync_timeline_get(struct sync_timeline *obj)
119{
120 kref_get(&obj->kref);
121}
122
123static void sync_timeline_put(struct sync_timeline *obj)
124{
125 kref_put(&obj->kref, sync_timeline_free);
126}
127
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300128static const char *timeline_fence_get_driver_name(struct fence *fence)
129{
Gustavo Padovanb9bc2b72016-05-31 16:59:11 -0300130 return "sw_sync";
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300131}
132
133static const char *timeline_fence_get_timeline_name(struct fence *fence)
134{
135 struct sync_timeline *parent = fence_parent(fence);
136
137 return parent->name;
138}
139
140static void timeline_fence_release(struct fence *fence)
141{
142 struct sync_pt *pt = fence_to_sync_pt(fence);
143 struct sync_timeline *parent = fence_parent(fence);
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300144
Gustavo Padovan4496b882017-07-29 12:22:15 -0300145 if (!list_empty(&pt->link)) {
146 unsigned long flags;
147
148 spin_lock_irqsave(fence->lock, flags);
149 if (!list_empty(&pt->link)) {
150 list_del(&pt->link);
151 rb_erase(&pt->node, &parent->pt_tree);
152 }
153 spin_unlock_irqrestore(fence->lock, flags);
154 }
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300155
156 sync_timeline_put(parent);
157 fence_free(fence);
158}
159
160static bool timeline_fence_signaled(struct fence *fence)
161{
162 struct sync_timeline *parent = fence_parent(fence);
163
Gustavo Padovan4496b882017-07-29 12:22:15 -0300164 return !__fence_is_later(fence->seqno, parent->value);
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300165}
166
167static bool timeline_fence_enable_signaling(struct fence *fence)
168{
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300169 return true;
170}
171
Dmitry Torokhov2a061a82015-09-08 17:30:52 -0700172static void timeline_fence_disable_signaling(struct fence *fence)
173{
174 struct sync_pt *pt = container_of(fence, struct sync_pt, base);
175
Amit Pundir2fffe2e2017-09-18 00:10:09 +0100176 list_del_init(&pt->link);
Dmitry Torokhov2a061a82015-09-08 17:30:52 -0700177}
178
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300179static void timeline_fence_value_str(struct fence *fence,
180 char *str, int size)
181{
182 snprintf(str, size, "%d", fence->seqno);
183}
184
185static void timeline_fence_timeline_value_str(struct fence *fence,
186 char *str, int size)
187{
188 struct sync_timeline *parent = fence_parent(fence);
189
190 snprintf(str, size, "%d", parent->value);
191}
192
193static const struct fence_ops timeline_fence_ops = {
194 .get_driver_name = timeline_fence_get_driver_name,
195 .get_timeline_name = timeline_fence_get_timeline_name,
196 .enable_signaling = timeline_fence_enable_signaling,
Dmitry Torokhov2a061a82015-09-08 17:30:52 -0700197 .disable_signaling = timeline_fence_disable_signaling,
Gustavo Padovanaff9da12016-05-31 16:59:07 -0300198 .signaled = timeline_fence_signaled,
199 .wait = fence_default_wait,
200 .release = timeline_fence_release,
201 .fence_value_str = timeline_fence_value_str,
202 .timeline_value_str = timeline_fence_timeline_value_str,
203};
204
Gustavo Padovan1867a232016-05-31 16:59:05 -0300205/**
206 * sync_timeline_signal() - signal a status change on a sync_timeline
207 * @obj: sync_timeline to signal
208 * @inc: num to increment on timeline->value
209 *
210 * A sync implementation should call this any time one of it's fences
211 * has signaled or has an error condition.
212 */
213static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
214{
Gustavo Padovan1867a232016-05-31 16:59:05 -0300215 struct sync_pt *pt, *next;
216
217 trace_sync_timeline(obj);
218
Chris Wilsone82ecb22017-06-29 22:05:32 +0100219 spin_lock_irq(&obj->lock);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300220
221 obj->value += inc;
222
Chris Wilsondb767402017-06-29 22:12:53 +0100223 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
Gustavo Padovan9ae85fa2017-07-29 12:22:16 -0300224 if (!timeline_fence_signaled(&pt->base))
Chris Wilsondb767402017-06-29 22:12:53 +0100225 break;
226
227 list_del_init(&pt->link);
228 rb_erase(&pt->node, &obj->pt_tree);
Gustavo Padovan9ae85fa2017-07-29 12:22:16 -0300229
230 /*
231 * A signal callback may release the last reference to this
232 * fence, causing it to be freed. That operation has to be
233 * last to avoid a use after free inside this loop, and must
234 * be after we remove the fence from the timeline in order to
235 * prevent deadlocking on timeline->lock inside
236 * timeline_fence_release().
237 */
238 fence_signal_locked(&pt->base);
Chris Wilsondb767402017-06-29 22:12:53 +0100239 }
Gustavo Padovan1867a232016-05-31 16:59:05 -0300240
Chris Wilsone82ecb22017-06-29 22:05:32 +0100241 spin_unlock_irq(&obj->lock);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300242}
243
244/**
245 * sync_pt_create() - creates a sync pt
246 * @parent: fence's parent sync_timeline
Gustavo Padovan1867a232016-05-31 16:59:05 -0300247 * @inc: value of the fence
248 *
249 * Creates a new sync_pt as a child of @parent. @size bytes will be
250 * allocated allowing for implementation specific data to be kept after
251 * the generic sync_timeline struct. Returns the sync_pt object or
252 * NULL in case of error.
253 */
Chris Wilson1bf0b232017-06-29 13:59:28 +0100254static struct sync_pt *sync_pt_create(struct sync_timeline *obj,
255 unsigned int value)
Gustavo Padovan1867a232016-05-31 16:59:05 -0300256{
Gustavo Padovan1867a232016-05-31 16:59:05 -0300257 struct sync_pt *pt;
258
Chris Wilson1bf0b232017-06-29 13:59:28 +0100259 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300260 if (!pt)
261 return NULL;
262
Gustavo Padovan1867a232016-05-31 16:59:05 -0300263 sync_timeline_get(obj);
Chris Wilsone82ecb22017-06-29 22:05:32 +0100264 fence_init(&pt->base, &timeline_fence_ops, &obj->lock,
Gustavo Padovan1867a232016-05-31 16:59:05 -0300265 obj->context, value);
Chris Wilsone82ecb22017-06-29 22:05:32 +0100266 INIT_LIST_HEAD(&pt->link);
Chris Wilsonf14ad422017-06-29 13:59:27 +0100267
Chris Wilsone82ecb22017-06-29 22:05:32 +0100268 spin_lock_irq(&obj->lock);
Chris Wilsondb767402017-06-29 22:12:53 +0100269 if (!fence_is_signaled_locked(&pt->base)) {
270 struct rb_node **p = &obj->pt_tree.rb_node;
271 struct rb_node *parent = NULL;
272
273 while (*p) {
274 struct sync_pt *other;
275 int cmp;
276
277 parent = *p;
278 other = rb_entry(parent, typeof(*pt), node);
279 cmp = value - other->base.seqno;
280 if (cmp > 0) {
281 p = &parent->rb_right;
282 } else if (cmp < 0) {
283 p = &parent->rb_left;
284 } else {
285 if (fence_get_rcu(&other->base)) {
286 fence_put(&pt->base);
287 pt = other;
288 goto unlock;
289 }
290 p = &parent->rb_left;
291 }
292 }
293 rb_link_node(&pt->node, parent, p);
294 rb_insert_color(&pt->node, &obj->pt_tree);
295
296 parent = rb_next(&pt->node);
297 list_add_tail(&pt->link,
298 parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list);
299 }
300unlock:
Chris Wilsone82ecb22017-06-29 22:05:32 +0100301 spin_unlock_irq(&obj->lock);
Chris Wilsonf14ad422017-06-29 13:59:27 +0100302
Gustavo Padovan1867a232016-05-31 16:59:05 -0300303 return pt;
304}
305
Gustavo Padovan1867a232016-05-31 16:59:05 -0300306/*
307 * *WARNING*
308 *
309 * improper use of this can result in deadlocking kernel drivers from userspace.
310 */
311
312/* opening sw_sync create a new sync obj */
313static int sw_sync_debugfs_open(struct inode *inode, struct file *file)
314{
315 struct sync_timeline *obj;
316 char task_comm[TASK_COMM_LEN];
317
318 get_task_comm(task_comm, current);
319
Gustavo Padovanb9bc2b72016-05-31 16:59:11 -0300320 obj = sync_timeline_create(task_comm);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300321 if (!obj)
322 return -ENOMEM;
323
324 file->private_data = obj;
325
326 return 0;
327}
328
329static int sw_sync_debugfs_release(struct inode *inode, struct file *file)
330{
331 struct sync_timeline *obj = file->private_data;
Dominik Behr424bdc52017-09-07 16:02:46 -0300332 struct sync_pt *pt, *next;
Gustavo Padovan1867a232016-05-31 16:59:05 -0300333
Dominik Behr424bdc52017-09-07 16:02:46 -0300334 spin_lock_irq(&obj->lock);
335
336 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
337 fence_set_error(&pt->base, -ENOENT);
338 fence_signal_locked(&pt->base);
339 }
340
341 spin_unlock_irq(&obj->lock);
Gustavo Padovan71110232016-05-31 16:59:10 -0300342
343 sync_timeline_put(obj);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300344 return 0;
345}
346
347static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,
348 unsigned long arg)
349{
350 int fd = get_unused_fd_flags(O_CLOEXEC);
351 int err;
352 struct sync_pt *pt;
353 struct sync_file *sync_file;
354 struct sw_sync_create_fence_data data;
355
356 if (fd < 0)
357 return fd;
358
359 if (copy_from_user(&data, (void __user *)arg, sizeof(data))) {
360 err = -EFAULT;
361 goto err;
362 }
363
Chris Wilson1bf0b232017-06-29 13:59:28 +0100364 pt = sync_pt_create(obj, data.value);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300365 if (!pt) {
366 err = -ENOMEM;
367 goto err;
368 }
369
370 sync_file = sync_file_create(&pt->base);
Gustavo Padovana2837742016-10-26 18:59:59 -0200371 fence_put(&pt->base);
Gustavo Padovan1867a232016-05-31 16:59:05 -0300372 if (!sync_file) {
Gustavo Padovan1867a232016-05-31 16:59:05 -0300373 err = -ENOMEM;
374 goto err;
375 }
376
377 data.fence = fd;
378 if (copy_to_user((void __user *)arg, &data, sizeof(data))) {
379 fput(sync_file->file);
380 err = -EFAULT;
381 goto err;
382 }
383
384 fd_install(fd, sync_file->file);
385
386 return 0;
387
388err:
389 put_unused_fd(fd);
390 return err;
391}
392
393static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg)
394{
395 u32 value;
396
397 if (copy_from_user(&value, (void __user *)arg, sizeof(value)))
398 return -EFAULT;
399
Chris Wilson985b5b22017-06-29 13:59:26 +0100400 while (value > INT_MAX) {
401 sync_timeline_signal(obj, INT_MAX);
402 value -= INT_MAX;
403 }
404
Gustavo Padovan1867a232016-05-31 16:59:05 -0300405 sync_timeline_signal(obj, value);
406
407 return 0;
408}
409
410static long sw_sync_ioctl(struct file *file, unsigned int cmd,
411 unsigned long arg)
412{
413 struct sync_timeline *obj = file->private_data;
414
415 switch (cmd) {
416 case SW_SYNC_IOC_CREATE_FENCE:
417 return sw_sync_ioctl_create_fence(obj, arg);
418
419 case SW_SYNC_IOC_INC:
420 return sw_sync_ioctl_inc(obj, arg);
421
422 default:
423 return -ENOTTY;
424 }
425}
426
427const struct file_operations sw_sync_debugfs_fops = {
428 .open = sw_sync_debugfs_open,
429 .release = sw_sync_debugfs_release,
430 .unlocked_ioctl = sw_sync_ioctl,
431 .compat_ioctl = sw_sync_ioctl,
432};