Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Fence mechanism for dma-buf to allow for asynchronous dma access |
| 3 | * |
| 4 | * Copyright (C) 2012 Canonical Ltd |
| 5 | * Copyright (C) 2012 Texas Instruments |
| 6 | * |
| 7 | * Authors: |
| 8 | * Rob Clark <robdclark@gmail.com> |
| 9 | * Maarten Lankhorst <maarten.lankhorst@canonical.com> |
| 10 | * |
| 11 | * This program is free software; you can redistribute it and/or modify it |
| 12 | * under the terms of the GNU General Public License version 2 as published by |
| 13 | * the Free Software Foundation. |
| 14 | * |
| 15 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 16 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 18 | * more details. |
| 19 | */ |
| 20 | |
| 21 | #ifndef __LINUX_FENCE_H |
| 22 | #define __LINUX_FENCE_H |
| 23 | |
| 24 | #include <linux/err.h> |
| 25 | #include <linux/wait.h> |
| 26 | #include <linux/list.h> |
| 27 | #include <linux/bitops.h> |
| 28 | #include <linux/kref.h> |
| 29 | #include <linux/sched.h> |
| 30 | #include <linux/printk.h> |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 31 | #include <linux/rcupdate.h> |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 32 | |
| 33 | struct fence; |
| 34 | struct fence_ops; |
| 35 | struct fence_cb; |
| 36 | |
| 37 | /** |
| 38 | * struct fence - software synchronization primitive |
| 39 | * @refcount: refcount for this fence |
| 40 | * @ops: fence_ops associated with this fence |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 41 | * @rcu: used for releasing fence with kfree_rcu |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 42 | * @cb_list: list of all callbacks to call |
| 43 | * @lock: spin_lock_irqsave used for locking |
| 44 | * @context: execution context this fence belongs to, returned by |
| 45 | * fence_context_alloc() |
| 46 | * @seqno: the sequence number of this fence inside the execution context, |
| 47 | * can be compared to decide which fence would be signaled later. |
| 48 | * @flags: A mask of FENCE_FLAG_* defined below |
| 49 | * @timestamp: Timestamp when the fence was signaled. |
| 50 | * @status: Optional, only valid if < 0, must be set before calling |
| 51 | * fence_signal, indicates that the fence has completed with an error. |
| 52 | * |
| 53 | * the flags member must be manipulated and read using the appropriate |
| 54 | * atomic ops (bit_*), so taking the spinlock will not be needed most |
| 55 | * of the time. |
| 56 | * |
| 57 | * FENCE_FLAG_SIGNALED_BIT - fence is already signaled |
| 58 | * FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called* |
| 59 | * FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the |
| 60 | * implementer of the fence for its own purposes. Can be used in different |
| 61 | * ways by different fence implementers, so do not rely on this. |
| 62 | * |
| 63 | * *) Since atomic bitops are used, this is not guaranteed to be the case. |
| 64 | * Particularly, if the bit was set, but fence_signal was called right |
| 65 | * before this bit was set, it would have been able to set the |
| 66 | * FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called. |
| 67 | * Adding a check for FENCE_FLAG_SIGNALED_BIT after setting |
| 68 | * FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that |
| 69 | * after fence_signal was called, any enable_signaling call will have either |
| 70 | * been completed, or never called at all. |
| 71 | */ |
| 72 | struct fence { |
| 73 | struct kref refcount; |
| 74 | const struct fence_ops *ops; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 75 | struct rcu_head rcu; |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 76 | struct list_head cb_list; |
| 77 | spinlock_t *lock; |
| 78 | unsigned context, seqno; |
| 79 | unsigned long flags; |
| 80 | ktime_t timestamp; |
| 81 | int status; |
| 82 | }; |
| 83 | |
| 84 | enum fence_flag_bits { |
| 85 | FENCE_FLAG_SIGNALED_BIT, |
| 86 | FENCE_FLAG_ENABLE_SIGNAL_BIT, |
| 87 | FENCE_FLAG_USER_BITS, /* must always be last member */ |
| 88 | }; |
| 89 | |
| 90 | typedef void (*fence_func_t)(struct fence *fence, struct fence_cb *cb); |
| 91 | |
| 92 | /** |
| 93 | * struct fence_cb - callback for fence_add_callback |
| 94 | * @node: used by fence_add_callback to append this struct to fence::cb_list |
| 95 | * @func: fence_func_t to call |
| 96 | * |
| 97 | * This struct will be initialized by fence_add_callback, additional |
| 98 | * data can be passed along by embedding fence_cb in another struct. |
| 99 | */ |
| 100 | struct fence_cb { |
| 101 | struct list_head node; |
| 102 | fence_func_t func; |
| 103 | }; |
| 104 | |
| 105 | /** |
| 106 | * struct fence_ops - operations implemented for fence |
| 107 | * @get_driver_name: returns the driver name. |
| 108 | * @get_timeline_name: return the name of the context this fence belongs to. |
| 109 | * @enable_signaling: enable software signaling of fence. |
| 110 | * @signaled: [optional] peek whether the fence is signaled, can be null. |
| 111 | * @wait: custom wait implementation, or fence_default_wait. |
| 112 | * @release: [optional] called on destruction of fence, can be null |
| 113 | * @fill_driver_data: [optional] callback to fill in free-form debug info |
| 114 | * Returns amount of bytes filled, or -errno. |
| 115 | * @fence_value_str: [optional] fills in the value of the fence as a string |
| 116 | * @timeline_value_str: [optional] fills in the current value of the timeline |
| 117 | * as a string |
| 118 | * |
| 119 | * Notes on enable_signaling: |
| 120 | * For fence implementations that have the capability for hw->hw |
| 121 | * signaling, they can implement this op to enable the necessary |
| 122 | * irqs, or insert commands into cmdstream, etc. This is called |
| 123 | * in the first wait() or add_callback() path to let the fence |
| 124 | * implementation know that there is another driver waiting on |
| 125 | * the signal (ie. hw->sw case). |
| 126 | * |
| 127 | * This function can be called called from atomic context, but not |
| 128 | * from irq context, so normal spinlocks can be used. |
| 129 | * |
| 130 | * A return value of false indicates the fence already passed, |
Masanari Iida | f353d71 | 2014-10-22 00:00:14 +0900 | [diff] [blame] | 131 | * or some failure occurred that made it impossible to enable |
| 132 | * signaling. True indicates successful enabling. |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 133 | * |
| 134 | * fence->status may be set in enable_signaling, but only when false is |
| 135 | * returned. |
| 136 | * |
| 137 | * Calling fence_signal before enable_signaling is called allows |
| 138 | * for a tiny race window in which enable_signaling is called during, |
| 139 | * before, or after fence_signal. To fight this, it is recommended |
| 140 | * that before enable_signaling returns true an extra reference is |
| 141 | * taken on the fence, to be released when the fence is signaled. |
| 142 | * This will mean fence_signal will still be called twice, but |
| 143 | * the second time will be a noop since it was already signaled. |
| 144 | * |
| 145 | * Notes on signaled: |
| 146 | * May set fence->status if returning true. |
| 147 | * |
| 148 | * Notes on wait: |
| 149 | * Must not be NULL, set to fence_default_wait for default implementation. |
| 150 | * the fence_default_wait implementation should work for any fence, as long |
| 151 | * as enable_signaling works correctly. |
| 152 | * |
| 153 | * Must return -ERESTARTSYS if the wait is intr = true and the wait was |
| 154 | * interrupted, and remaining jiffies if fence has signaled, or 0 if wait |
| 155 | * timed out. Can also return other error values on custom implementations, |
| 156 | * which should be treated as if the fence is signaled. For example a hardware |
| 157 | * lockup could be reported like that. |
| 158 | * |
| 159 | * Notes on release: |
| 160 | * Can be NULL, this function allows additional commands to run on |
| 161 | * destruction of the fence. Can be called from irq context. |
| 162 | * If pointer is set to NULL, kfree will get called instead. |
| 163 | */ |
| 164 | |
| 165 | struct fence_ops { |
| 166 | const char * (*get_driver_name)(struct fence *fence); |
| 167 | const char * (*get_timeline_name)(struct fence *fence); |
| 168 | bool (*enable_signaling)(struct fence *fence); |
| 169 | bool (*signaled)(struct fence *fence); |
| 170 | signed long (*wait)(struct fence *fence, bool intr, signed long timeout); |
| 171 | void (*release)(struct fence *fence); |
| 172 | |
| 173 | int (*fill_driver_data)(struct fence *fence, void *data, int size); |
| 174 | void (*fence_value_str)(struct fence *fence, char *str, int size); |
| 175 | void (*timeline_value_str)(struct fence *fence, char *str, int size); |
| 176 | }; |
| 177 | |
| 178 | void fence_init(struct fence *fence, const struct fence_ops *ops, |
| 179 | spinlock_t *lock, unsigned context, unsigned seqno); |
| 180 | |
| 181 | void fence_release(struct kref *kref); |
| 182 | void fence_free(struct fence *fence); |
| 183 | |
| 184 | /** |
| 185 | * fence_get - increases refcount of the fence |
| 186 | * @fence: [in] fence to increase refcount of |
| 187 | * |
| 188 | * Returns the same fence, with refcount increased by 1. |
| 189 | */ |
| 190 | static inline struct fence *fence_get(struct fence *fence) |
| 191 | { |
| 192 | if (fence) |
| 193 | kref_get(&fence->refcount); |
| 194 | return fence; |
| 195 | } |
| 196 | |
| 197 | /** |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 198 | * fence_get_rcu - get a fence from a reservation_object_list with rcu read lock |
| 199 | * @fence: [in] fence to increase refcount of |
| 200 | * |
| 201 | * Function returns NULL if no refcount could be obtained, or the fence. |
| 202 | */ |
| 203 | static inline struct fence *fence_get_rcu(struct fence *fence) |
| 204 | { |
| 205 | if (kref_get_unless_zero(&fence->refcount)) |
| 206 | return fence; |
| 207 | else |
| 208 | return NULL; |
| 209 | } |
| 210 | |
| 211 | /** |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 212 | * fence_put - decreases refcount of the fence |
| 213 | * @fence: [in] fence to reduce refcount of |
| 214 | */ |
| 215 | static inline void fence_put(struct fence *fence) |
| 216 | { |
| 217 | if (fence) |
| 218 | kref_put(&fence->refcount, fence_release); |
| 219 | } |
| 220 | |
| 221 | int fence_signal(struct fence *fence); |
| 222 | int fence_signal_locked(struct fence *fence); |
| 223 | signed long fence_default_wait(struct fence *fence, bool intr, signed long timeout); |
| 224 | int fence_add_callback(struct fence *fence, struct fence_cb *cb, |
| 225 | fence_func_t func); |
| 226 | bool fence_remove_callback(struct fence *fence, struct fence_cb *cb); |
| 227 | void fence_enable_sw_signaling(struct fence *fence); |
| 228 | |
| 229 | /** |
| 230 | * fence_is_signaled_locked - Return an indication if the fence is signaled yet. |
| 231 | * @fence: [in] the fence to check |
| 232 | * |
| 233 | * Returns true if the fence was already signaled, false if not. Since this |
| 234 | * function doesn't enable signaling, it is not guaranteed to ever return |
| 235 | * true if fence_add_callback, fence_wait or fence_enable_sw_signaling |
| 236 | * haven't been called before. |
| 237 | * |
| 238 | * This function requires fence->lock to be held. |
| 239 | */ |
| 240 | static inline bool |
| 241 | fence_is_signaled_locked(struct fence *fence) |
| 242 | { |
| 243 | if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) |
| 244 | return true; |
| 245 | |
| 246 | if (fence->ops->signaled && fence->ops->signaled(fence)) { |
| 247 | fence_signal_locked(fence); |
| 248 | return true; |
| 249 | } |
| 250 | |
| 251 | return false; |
| 252 | } |
| 253 | |
| 254 | /** |
| 255 | * fence_is_signaled - Return an indication if the fence is signaled yet. |
| 256 | * @fence: [in] the fence to check |
| 257 | * |
| 258 | * Returns true if the fence was already signaled, false if not. Since this |
| 259 | * function doesn't enable signaling, it is not guaranteed to ever return |
| 260 | * true if fence_add_callback, fence_wait or fence_enable_sw_signaling |
| 261 | * haven't been called before. |
| 262 | * |
| 263 | * It's recommended for seqno fences to call fence_signal when the |
| 264 | * operation is complete, it makes it possible to prevent issues from |
| 265 | * wraparound between time of issue and time of use by checking the return |
| 266 | * value of this function before calling hardware-specific wait instructions. |
| 267 | */ |
| 268 | static inline bool |
| 269 | fence_is_signaled(struct fence *fence) |
| 270 | { |
| 271 | if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) |
| 272 | return true; |
| 273 | |
| 274 | if (fence->ops->signaled && fence->ops->signaled(fence)) { |
| 275 | fence_signal(fence); |
| 276 | return true; |
| 277 | } |
| 278 | |
| 279 | return false; |
| 280 | } |
| 281 | |
| 282 | /** |
Christian König | 6c455ac | 2015-10-21 12:58:17 +0200 | [diff] [blame] | 283 | * fence_is_later - return if f1 is chronologically later than f2 |
| 284 | * @f1: [in] the first fence from the same context |
| 285 | * @f2: [in] the second fence from the same context |
| 286 | * |
| 287 | * Returns true if f1 is chronologically later than f2. Both fences must be |
| 288 | * from the same context, since a seqno is not re-used across contexts. |
| 289 | */ |
| 290 | static inline bool fence_is_later(struct fence *f1, struct fence *f2) |
| 291 | { |
| 292 | if (WARN_ON(f1->context != f2->context)) |
| 293 | return false; |
| 294 | |
Christian König | b47bcb93 | 2016-02-29 10:53:11 -0500 | [diff] [blame^] | 295 | return (int)(f1->seqno - f2->seqno) > 0; |
Christian König | 6c455ac | 2015-10-21 12:58:17 +0200 | [diff] [blame] | 296 | } |
| 297 | |
| 298 | /** |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 299 | * fence_later - return the chronologically later fence |
| 300 | * @f1: [in] the first fence from the same context |
| 301 | * @f2: [in] the second fence from the same context |
| 302 | * |
| 303 | * Returns NULL if both fences are signaled, otherwise the fence that would be |
| 304 | * signaled last. Both fences must be from the same context, since a seqno is |
| 305 | * not re-used across contexts. |
| 306 | */ |
| 307 | static inline struct fence *fence_later(struct fence *f1, struct fence *f2) |
| 308 | { |
| 309 | if (WARN_ON(f1->context != f2->context)) |
| 310 | return NULL; |
| 311 | |
| 312 | /* |
| 313 | * can't check just FENCE_FLAG_SIGNALED_BIT here, it may never have been |
| 314 | * set if enable_signaling wasn't called, and enabling that here is |
| 315 | * overkill. |
| 316 | */ |
Christian König | 6c455ac | 2015-10-21 12:58:17 +0200 | [diff] [blame] | 317 | if (fence_is_later(f1, f2)) |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 318 | return fence_is_signaled(f1) ? NULL : f1; |
Christian König | 6c455ac | 2015-10-21 12:58:17 +0200 | [diff] [blame] | 319 | else |
| 320 | return fence_is_signaled(f2) ? NULL : f2; |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 321 | } |
| 322 | |
| 323 | signed long fence_wait_timeout(struct fence *, bool intr, signed long timeout); |
Christian König | a519435 | 2015-10-20 16:34:16 +0200 | [diff] [blame] | 324 | signed long fence_wait_any_timeout(struct fence **fences, uint32_t count, |
| 325 | bool intr, signed long timeout); |
Maarten Lankhorst | e941759 | 2014-07-01 12:57:14 +0200 | [diff] [blame] | 326 | |
| 327 | /** |
| 328 | * fence_wait - sleep until the fence gets signaled |
| 329 | * @fence: [in] the fence to wait on |
| 330 | * @intr: [in] if true, do an interruptible wait |
| 331 | * |
| 332 | * This function will return -ERESTARTSYS if interrupted by a signal, |
| 333 | * or 0 if the fence was signaled. Other error values may be |
| 334 | * returned on custom implementations. |
| 335 | * |
| 336 | * Performs a synchronous wait on this fence. It is assumed the caller |
| 337 | * directly or indirectly holds a reference to the fence, otherwise the |
| 338 | * fence might be freed before return, resulting in undefined behavior. |
| 339 | */ |
| 340 | static inline signed long fence_wait(struct fence *fence, bool intr) |
| 341 | { |
| 342 | signed long ret; |
| 343 | |
| 344 | /* Since fence_wait_timeout cannot timeout with |
| 345 | * MAX_SCHEDULE_TIMEOUT, only valid return values are |
| 346 | * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT. |
| 347 | */ |
| 348 | ret = fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT); |
| 349 | |
| 350 | return ret < 0 ? ret : 0; |
| 351 | } |
| 352 | |
| 353 | unsigned fence_context_alloc(unsigned num); |
| 354 | |
| 355 | #define FENCE_TRACE(f, fmt, args...) \ |
| 356 | do { \ |
| 357 | struct fence *__ff = (f); \ |
| 358 | if (config_enabled(CONFIG_FENCE_TRACE)) \ |
| 359 | pr_info("f %u#%u: " fmt, \ |
| 360 | __ff->context, __ff->seqno, ##args); \ |
| 361 | } while (0) |
| 362 | |
| 363 | #define FENCE_WARN(f, fmt, args...) \ |
| 364 | do { \ |
| 365 | struct fence *__ff = (f); \ |
| 366 | pr_warn("f %u#%u: " fmt, __ff->context, __ff->seqno, \ |
| 367 | ##args); \ |
| 368 | } while (0) |
| 369 | |
| 370 | #define FENCE_ERR(f, fmt, args...) \ |
| 371 | do { \ |
| 372 | struct fence *__ff = (f); \ |
| 373 | pr_err("f %u#%u: " fmt, __ff->context, __ff->seqno, \ |
| 374 | ##args); \ |
| 375 | } while (0) |
| 376 | |
| 377 | #endif /* __LINUX_FENCE_H */ |