| /************************************************************************** |
| * |
| * Copyright © 2011 VMware, Inc., Palo Alto, CA., USA |
| * All Rights Reserved. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the |
| * "Software"), to deal in the Software without restriction, including |
| * without limitation the rights to use, copy, modify, merge, publish, |
| * distribute, sub license, and/or sell copies of the Software, and to |
| * permit persons to whom the Software is furnished to do so, subject to |
| * the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the |
| * next paragraph) shall be included in all copies or substantial portions |
| * of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
| * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
| * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
| * USE OR OTHER DEALINGS IN THE SOFTWARE. |
| * |
| **************************************************************************/ |
| |
| #include "drmP.h" |
| #include "vmwgfx_drv.h" |
| |
| #define VMW_FENCE_WRAP (1 << 31) |
| |
| struct vmw_fence_manager { |
| int num_fence_objects; |
| struct vmw_private *dev_priv; |
| spinlock_t lock; |
| u32 next_seqno; |
| struct list_head fence_list; |
| struct work_struct work; |
| u32 user_fence_size; |
| u32 fence_size; |
| bool fifo_down; |
| struct list_head cleanup_list; |
| }; |
| |
| struct vmw_user_fence { |
| struct ttm_base_object base; |
| struct vmw_fence_obj fence; |
| }; |
| |
| /** |
| * vmw_fence_destroy_locked |
| * |
| */ |
| |
| static void vmw_fence_obj_destroy_locked(struct kref *kref) |
| { |
| struct vmw_fence_obj *fence = |
| container_of(kref, struct vmw_fence_obj, kref); |
| |
| struct vmw_fence_manager *fman = fence->fman; |
| unsigned int num_fences; |
| |
| list_del_init(&fence->head); |
| num_fences = --fman->num_fence_objects; |
| spin_unlock_irq(&fman->lock); |
| if (fence->destroy) |
| fence->destroy(fence); |
| else |
| kfree(fence); |
| |
| spin_lock_irq(&fman->lock); |
| } |
| |
| |
| /** |
| * Execute signal actions on fences recently signaled. |
| * This is done from a workqueue so we don't have to execute |
| * signal actions from atomic context. |
| */ |
| |
| static void vmw_fence_work_func(struct work_struct *work) |
| { |
| struct vmw_fence_manager *fman = |
| container_of(work, struct vmw_fence_manager, work); |
| struct list_head list; |
| struct vmw_fence_action *action, *next_action; |
| |
| do { |
| INIT_LIST_HEAD(&list); |
| spin_lock_irq(&fman->lock); |
| list_splice_init(&fman->cleanup_list, &list); |
| spin_unlock_irq(&fman->lock); |
| |
| if (list_empty(&list)) |
| return; |
| |
| /* |
| * At this point, only we should be able to manipulate the |
| * list heads of the actions we have on the private list. |
| */ |
| |
| list_for_each_entry_safe(action, next_action, &list, head) { |
| list_del_init(&action->head); |
| action->cleanup(action); |
| } |
| } while (1); |
| } |
| |
| struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv) |
| { |
| struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL); |
| |
| if (unlikely(fman == NULL)) |
| return NULL; |
| |
| fman->dev_priv = dev_priv; |
| spin_lock_init(&fman->lock); |
| INIT_LIST_HEAD(&fman->fence_list); |
| INIT_LIST_HEAD(&fman->cleanup_list); |
| INIT_WORK(&fman->work, &vmw_fence_work_func); |
| fman->fifo_down = true; |
| fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence)); |
| fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj)); |
| |
| return fman; |
| } |
| |
| void vmw_fence_manager_takedown(struct vmw_fence_manager *fman) |
| { |
| unsigned long irq_flags; |
| bool lists_empty; |
| |
| (void) cancel_work_sync(&fman->work); |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| lists_empty = list_empty(&fman->fence_list) && |
| list_empty(&fman->cleanup_list); |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| |
| BUG_ON(!lists_empty); |
| kfree(fman); |
| } |
| |
| static int vmw_fence_obj_init(struct vmw_fence_manager *fman, |
| struct vmw_fence_obj *fence, |
| u32 seqno, |
| uint32_t mask, |
| void (*destroy) (struct vmw_fence_obj *fence)) |
| { |
| unsigned long irq_flags; |
| unsigned int num_fences; |
| int ret = 0; |
| |
| fence->seqno = seqno; |
| INIT_LIST_HEAD(&fence->seq_passed_actions); |
| fence->fman = fman; |
| fence->signaled = 0; |
| fence->signal_mask = mask; |
| kref_init(&fence->kref); |
| fence->destroy = destroy; |
| init_waitqueue_head(&fence->queue); |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| if (unlikely(fman->fifo_down)) { |
| ret = -EBUSY; |
| goto out_unlock; |
| } |
| list_add_tail(&fence->head, &fman->fence_list); |
| num_fences = ++fman->num_fence_objects; |
| |
| out_unlock: |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| return ret; |
| |
| } |
| |
| struct vmw_fence_obj *vmw_fence_obj_reference(struct vmw_fence_obj *fence) |
| { |
| kref_get(&fence->kref); |
| return fence; |
| } |
| |
| /** |
| * vmw_fence_obj_unreference |
| * |
| * Note that this function may not be entered with disabled irqs since |
| * it may re-enable them in the destroy function. |
| * |
| */ |
| void vmw_fence_obj_unreference(struct vmw_fence_obj **fence_p) |
| { |
| struct vmw_fence_obj *fence = *fence_p; |
| struct vmw_fence_manager *fman = fence->fman; |
| |
| *fence_p = NULL; |
| spin_lock_irq(&fman->lock); |
| BUG_ON(atomic_read(&fence->kref.refcount) == 0); |
| kref_put(&fence->kref, vmw_fence_obj_destroy_locked); |
| spin_unlock_irq(&fman->lock); |
| } |
| |
| void vmw_fences_perform_actions(struct vmw_fence_manager *fman, |
| struct list_head *list) |
| { |
| struct vmw_fence_action *action, *next_action; |
| |
| list_for_each_entry_safe(action, next_action, list, head) { |
| list_del_init(&action->head); |
| if (action->seq_passed != NULL) |
| action->seq_passed(action); |
| |
| /* |
| * Add the cleanup action to the cleanup list so that |
| * it will be performed by a worker task. |
| */ |
| |
| if (action->cleanup != NULL) |
| list_add_tail(&action->head, &fman->cleanup_list); |
| } |
| } |
| |
| void vmw_fences_update(struct vmw_fence_manager *fman, u32 seqno) |
| { |
| unsigned long flags; |
| struct vmw_fence_obj *fence, *next_fence; |
| struct list_head action_list; |
| |
| spin_lock_irqsave(&fman->lock, flags); |
| list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) { |
| if (seqno - fence->seqno < VMW_FENCE_WRAP) { |
| list_del_init(&fence->head); |
| fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; |
| INIT_LIST_HEAD(&action_list); |
| list_splice_init(&fence->seq_passed_actions, |
| &action_list); |
| vmw_fences_perform_actions(fman, &action_list); |
| wake_up_all(&fence->queue); |
| } |
| |
| } |
| if (!list_empty(&fman->cleanup_list)) |
| (void) schedule_work(&fman->work); |
| spin_unlock_irqrestore(&fman->lock, flags); |
| } |
| |
| |
| bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence, |
| uint32_t flags) |
| { |
| struct vmw_fence_manager *fman = fence->fman; |
| unsigned long irq_flags; |
| uint32_t signaled; |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| signaled = fence->signaled; |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| |
| flags &= fence->signal_mask; |
| if ((signaled & flags) == flags) |
| return 1; |
| |
| if ((signaled & DRM_VMW_FENCE_FLAG_EXEC) == 0) { |
| struct vmw_private *dev_priv = fman->dev_priv; |
| __le32 __iomem *fifo_mem = dev_priv->mmio_virt; |
| u32 seqno; |
| |
| seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE); |
| vmw_fences_update(fman, seqno); |
| } |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| signaled = fence->signaled; |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| |
| return ((signaled & flags) == flags); |
| } |
| |
| int vmw_fence_obj_wait(struct vmw_fence_obj *fence, |
| uint32_t flags, bool lazy, |
| bool interruptible, unsigned long timeout) |
| { |
| struct vmw_private *dev_priv = fence->fman->dev_priv; |
| long ret; |
| |
| if (likely(vmw_fence_obj_signaled(fence, flags))) |
| return 0; |
| |
| vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC); |
| vmw_seqno_waiter_add(dev_priv); |
| |
| if (interruptible) |
| ret = wait_event_interruptible_timeout |
| (fence->queue, |
| vmw_fence_obj_signaled(fence, flags), |
| timeout); |
| else |
| ret = wait_event_timeout |
| (fence->queue, |
| vmw_fence_obj_signaled(fence, flags), |
| timeout); |
| |
| vmw_seqno_waiter_remove(dev_priv); |
| |
| if (unlikely(ret == 0)) |
| ret = -EBUSY; |
| else if (likely(ret > 0)) |
| ret = 0; |
| |
| return ret; |
| } |
| |
| void vmw_fence_obj_flush(struct vmw_fence_obj *fence) |
| { |
| struct vmw_private *dev_priv = fence->fman->dev_priv; |
| |
| vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC); |
| } |
| |
| static void vmw_fence_destroy(struct vmw_fence_obj *fence) |
| { |
| struct vmw_fence_manager *fman = fence->fman; |
| |
| kfree(fence); |
| /* |
| * Free kernel space accounting. |
| */ |
| ttm_mem_global_free(vmw_mem_glob(fman->dev_priv), |
| fman->fence_size); |
| } |
| |
| int vmw_fence_create(struct vmw_fence_manager *fman, |
| uint32_t seqno, |
| uint32_t mask, |
| struct vmw_fence_obj **p_fence) |
| { |
| struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv); |
| struct vmw_fence_obj *fence; |
| int ret; |
| |
| ret = ttm_mem_global_alloc(mem_glob, fman->fence_size, |
| false, false); |
| if (unlikely(ret != 0)) |
| return ret; |
| |
| fence = kzalloc(sizeof(*fence), GFP_KERNEL); |
| if (unlikely(fence == NULL)) { |
| ret = -ENOMEM; |
| goto out_no_object; |
| } |
| |
| ret = vmw_fence_obj_init(fman, fence, seqno, mask, |
| vmw_fence_destroy); |
| if (unlikely(ret != 0)) |
| goto out_err_init; |
| |
| *p_fence = fence; |
| return 0; |
| |
| out_err_init: |
| kfree(fence); |
| out_no_object: |
| ttm_mem_global_free(mem_glob, fman->fence_size); |
| return ret; |
| } |
| |
| |
| static void vmw_user_fence_destroy(struct vmw_fence_obj *fence) |
| { |
| struct vmw_user_fence *ufence = |
| container_of(fence, struct vmw_user_fence, fence); |
| struct vmw_fence_manager *fman = fence->fman; |
| |
| kfree(ufence); |
| /* |
| * Free kernel space accounting. |
| */ |
| ttm_mem_global_free(vmw_mem_glob(fman->dev_priv), |
| fman->user_fence_size); |
| } |
| |
| static void vmw_user_fence_base_release(struct ttm_base_object **p_base) |
| { |
| struct ttm_base_object *base = *p_base; |
| struct vmw_user_fence *ufence = |
| container_of(base, struct vmw_user_fence, base); |
| struct vmw_fence_obj *fence = &ufence->fence; |
| |
| *p_base = NULL; |
| vmw_fence_obj_unreference(&fence); |
| } |
| |
| int vmw_user_fence_create(struct drm_file *file_priv, |
| struct vmw_fence_manager *fman, |
| uint32_t seqno, |
| uint32_t mask, |
| struct vmw_fence_obj **p_fence, |
| uint32_t *p_handle) |
| { |
| struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; |
| struct vmw_user_fence *ufence; |
| struct vmw_fence_obj *tmp; |
| struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv); |
| int ret; |
| |
| /* |
| * Kernel memory space accounting, since this object may |
| * be created by a user-space request. |
| */ |
| |
| ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size, |
| false, false); |
| if (unlikely(ret != 0)) |
| return ret; |
| |
| ufence = kzalloc(sizeof(*ufence), GFP_KERNEL); |
| if (unlikely(ufence == NULL)) { |
| ret = -ENOMEM; |
| goto out_no_object; |
| } |
| |
| ret = vmw_fence_obj_init(fman, &ufence->fence, seqno, |
| mask, vmw_user_fence_destroy); |
| if (unlikely(ret != 0)) { |
| kfree(ufence); |
| goto out_no_object; |
| } |
| |
| /* |
| * The base object holds a reference which is freed in |
| * vmw_user_fence_base_release. |
| */ |
| tmp = vmw_fence_obj_reference(&ufence->fence); |
| ret = ttm_base_object_init(tfile, &ufence->base, false, |
| VMW_RES_FENCE, |
| &vmw_user_fence_base_release, NULL); |
| |
| |
| if (unlikely(ret != 0)) { |
| /* |
| * Free the base object's reference |
| */ |
| vmw_fence_obj_unreference(&tmp); |
| goto out_err; |
| } |
| |
| *p_fence = &ufence->fence; |
| *p_handle = ufence->base.hash.key; |
| |
| return 0; |
| out_err: |
| tmp = &ufence->fence; |
| vmw_fence_obj_unreference(&tmp); |
| out_no_object: |
| ttm_mem_global_free(mem_glob, fman->user_fence_size); |
| return ret; |
| } |
| |
| |
| /** |
| * vmw_fence_fifo_down - signal all unsignaled fence objects. |
| */ |
| |
| void vmw_fence_fifo_down(struct vmw_fence_manager *fman) |
| { |
| unsigned long irq_flags; |
| struct list_head action_list; |
| int ret; |
| |
| /* |
| * The list may be altered while we traverse it, so always |
| * restart when we've released the fman->lock. |
| */ |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| fman->fifo_down = true; |
| while (!list_empty(&fman->fence_list)) { |
| struct vmw_fence_obj *fence = |
| list_entry(fman->fence_list.prev, struct vmw_fence_obj, |
| head); |
| kref_get(&fence->kref); |
| spin_unlock_irq(&fman->lock); |
| |
| ret = vmw_fence_obj_wait(fence, fence->signal_mask, |
| false, false, |
| VMW_FENCE_WAIT_TIMEOUT); |
| |
| if (unlikely(ret != 0)) { |
| list_del_init(&fence->head); |
| fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; |
| INIT_LIST_HEAD(&action_list); |
| list_splice_init(&fence->seq_passed_actions, |
| &action_list); |
| vmw_fences_perform_actions(fman, &action_list); |
| wake_up_all(&fence->queue); |
| } |
| |
| spin_lock_irq(&fman->lock); |
| |
| BUG_ON(!list_empty(&fence->head)); |
| kref_put(&fence->kref, vmw_fence_obj_destroy_locked); |
| } |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| } |
| |
| void vmw_fence_fifo_up(struct vmw_fence_manager *fman) |
| { |
| unsigned long irq_flags; |
| |
| spin_lock_irqsave(&fman->lock, irq_flags); |
| fman->fifo_down = false; |
| spin_unlock_irqrestore(&fman->lock, irq_flags); |
| } |
| |
| |
| int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data, |
| struct drm_file *file_priv) |
| { |
| struct drm_vmw_fence_wait_arg *arg = |
| (struct drm_vmw_fence_wait_arg *)data; |
| unsigned long timeout; |
| struct ttm_base_object *base; |
| struct vmw_fence_obj *fence; |
| struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; |
| int ret; |
| uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ); |
| |
| /* |
| * 64-bit division not present on 32-bit systems, so do an |
| * approximation. (Divide by 1000000). |
| */ |
| |
| wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) - |
| (wait_timeout >> 26); |
| |
| if (!arg->cookie_valid) { |
| arg->cookie_valid = 1; |
| arg->kernel_cookie = jiffies + wait_timeout; |
| } |
| |
| base = ttm_base_object_lookup(tfile, arg->handle); |
| if (unlikely(base == NULL)) { |
| printk(KERN_ERR "Wait invalid fence object handle " |
| "0x%08lx.\n", |
| (unsigned long)arg->handle); |
| return -EINVAL; |
| } |
| |
| fence = &(container_of(base, struct vmw_user_fence, base)->fence); |
| |
| timeout = jiffies; |
| if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) { |
| ret = ((vmw_fence_obj_signaled(fence, arg->flags)) ? |
| 0 : -EBUSY); |
| goto out; |
| } |
| |
| timeout = (unsigned long)arg->kernel_cookie - timeout; |
| |
| ret = vmw_fence_obj_wait(fence, arg->flags, arg->lazy, true, timeout); |
| |
| out: |
| ttm_base_object_unref(&base); |
| |
| /* |
| * Optionally unref the fence object. |
| */ |
| |
| if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF)) |
| return ttm_ref_object_base_unref(tfile, arg->handle, |
| TTM_REF_USAGE); |
| return ret; |
| } |
| |
| int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data, |
| struct drm_file *file_priv) |
| { |
| struct drm_vmw_fence_signaled_arg *arg = |
| (struct drm_vmw_fence_signaled_arg *) data; |
| struct ttm_base_object *base; |
| struct vmw_fence_obj *fence; |
| struct vmw_fence_manager *fman; |
| struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; |
| struct vmw_private *dev_priv = vmw_priv(dev); |
| |
| base = ttm_base_object_lookup(tfile, arg->handle); |
| if (unlikely(base == NULL)) { |
| printk(KERN_ERR "Fence signaled invalid fence object handle " |
| "0x%08lx.\n", |
| (unsigned long)arg->handle); |
| return -EINVAL; |
| } |
| |
| fence = &(container_of(base, struct vmw_user_fence, base)->fence); |
| fman = fence->fman; |
| |
| arg->signaled = vmw_fence_obj_signaled(fence, arg->flags); |
| spin_lock_irq(&fman->lock); |
| |
| arg->signaled_flags = fence->signaled; |
| arg->passed_seqno = dev_priv->last_read_seqno; |
| spin_unlock_irq(&fman->lock); |
| |
| ttm_base_object_unref(&base); |
| |
| return 0; |
| } |
| |
| |
| int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data, |
| struct drm_file *file_priv) |
| { |
| struct drm_vmw_fence_arg *arg = |
| (struct drm_vmw_fence_arg *) data; |
| |
| return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile, |
| arg->handle, |
| TTM_REF_USAGE); |
| } |