drm/amdgpu: use a sync object for VMID fences v2
v2: rebase & cleanup
This way we can store more than one fence as user for each VMID.
Signed-off-by: Christian König <christian.koenig@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com> (v1)
Reviewed-by: Chunming Zhou <david1.zhou@amd.com> (v1)
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index c1b1004..148e2c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -588,6 +588,9 @@
struct amdgpu_sync *sync,
struct reservation_object *resv,
void *owner);
+bool amdgpu_sync_is_idle(struct amdgpu_sync *sync);
+int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
+ struct fence *fence);
struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
int amdgpu_sync_wait(struct amdgpu_sync *sync);
void amdgpu_sync_free(struct amdgpu_sync *sync);
@@ -875,7 +878,8 @@
struct amdgpu_vm_id {
struct list_head list;
- struct fence *active;
+ struct fence *first;
+ struct amdgpu_sync active;
atomic_long_t owner;
uint64_t pd_gpu_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c48b4fc..34a9280 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -109,6 +109,29 @@
}
/**
+ * amdgpu_sync_add_later - add the fence to the hash
+ *
+ * @sync: sync object to add the fence to
+ * @f: fence to add
+ *
+ * Tries to add the fence to an existing hash entry. Returns true when an entry
+ * was found, false otherwise.
+ */
+static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct fence *f)
+{
+ struct amdgpu_sync_entry *e;
+
+ hash_for_each_possible(sync->fences, e, node, f->context) {
+ if (unlikely(e->fence->context != f->context))
+ continue;
+
+ amdgpu_sync_keep_later(&e->fence, f);
+ return true;
+ }
+ return false;
+}
+
+/**
* amdgpu_sync_fence - remember to sync to this fence
*
* @sync: sync object to add fence to
@@ -127,13 +150,8 @@
amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM)
amdgpu_sync_keep_later(&sync->last_vm_update, f);
- hash_for_each_possible(sync->fences, e, node, f->context) {
- if (unlikely(e->fence->context != f->context))
- continue;
-
- amdgpu_sync_keep_later(&e->fence, f);
+ if (amdgpu_sync_add_later(sync, f))
return 0;
- }
e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
if (!e)
@@ -204,6 +222,81 @@
return r;
}
+/**
+ * amdgpu_sync_is_idle - test if all fences are signaled
+ *
+ * @sync: the sync object
+ *
+ * Returns true if all fences in the sync object are signaled.
+ */
+bool amdgpu_sync_is_idle(struct amdgpu_sync *sync)
+{
+ struct amdgpu_sync_entry *e;
+ struct hlist_node *tmp;
+ int i;
+
+ hash_for_each_safe(sync->fences, i, tmp, e, node) {
+ struct fence *f = e->fence;
+
+ if (fence_is_signaled(f)) {
+ hash_del(&e->node);
+ fence_put(f);
+ kmem_cache_free(amdgpu_sync_slab, e);
+ continue;
+ }
+
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * amdgpu_sync_cycle_fences - move fences from one sync object into another
+ *
+ * @dst: the destination sync object
+ * @src: the source sync object
+ * @fence: fence to add to source
+ *
+ * Remove all fences from source and put them into destination and add
+ * fence as new one into source.
+ */
+int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
+ struct fence *fence)
+{
+ struct amdgpu_sync_entry *e, *newone;
+ struct hlist_node *tmp;
+ int i;
+
+ /* Allocate the new entry before moving the old ones */
+ newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
+ if (!newone)
+ return -ENOMEM;
+
+ hash_for_each_safe(src->fences, i, tmp, e, node) {
+ struct fence *f = e->fence;
+
+ hash_del(&e->node);
+ if (fence_is_signaled(f)) {
+ fence_put(f);
+ kmem_cache_free(amdgpu_sync_slab, e);
+ continue;
+ }
+
+ if (amdgpu_sync_add_later(dst, f)) {
+ kmem_cache_free(amdgpu_sync_slab, e);
+ continue;
+ }
+
+ hash_add(dst->fences, &e->node, f->context);
+ }
+
+ hash_add(src->fences, &newone->node, fence->context);
+ newone->fence = fence_get(fence);
+
+ return 0;
+}
+
struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
{
struct amdgpu_sync_entry *e;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 8a758b4..d0cce7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -189,14 +189,13 @@
if (usable) {
- r = amdgpu_sync_fence(ring->adev, sync, id->active);
- if (r) {
- mutex_unlock(&adev->vm_manager.lock);
- return r;
- }
+ r = amdgpu_sync_fence(ring->adev, sync, id->first);
+ if (r)
+ goto error;
- fence_put(id->active);
- id->active = fence_get(fence);
+ r = amdgpu_sync_fence(ring->adev, &id->active, fence);
+ if (r)
+ goto error;
list_move_tail(&id->list, &adev->vm_manager.ids_lru);
@@ -214,13 +213,13 @@
struct amdgpu_vm_id,
list);
- if (id->active && !fence_is_signaled(id->active)) {
- struct amdgpu_vm_id *tmp;
+ if (!amdgpu_sync_is_idle(&id->active)) {
struct list_head *head = &adev->vm_manager.ids_lru;
+ struct amdgpu_vm_id *tmp;
list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru,
list) {
- if (id->active && fence_is_signaled(id->active)) {
+ if (amdgpu_sync_is_idle(&id->active)) {
list_move(&id->list, head);
head = &id->list;
}
@@ -230,25 +229,27 @@
list);
}
- r = amdgpu_sync_fence(ring->adev, sync, id->active);
- if (!r) {
- fence_put(id->active);
- id->active = fence_get(fence);
+ r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
+ if (r)
+ goto error;
- fence_put(id->flushed_updates);
- id->flushed_updates = fence_get(updates);
+ fence_put(id->first);
+ id->first = fence_get(fence);
- id->pd_gpu_addr = pd_addr;
+ fence_put(id->flushed_updates);
+ id->flushed_updates = fence_get(updates);
- list_move_tail(&id->list, &adev->vm_manager.ids_lru);
- atomic_long_set(&id->owner, (long)&vm->ids[ring->idx]);
- vm->ids[ring->idx] = id;
+ id->pd_gpu_addr = pd_addr;
- *vm_id = id - adev->vm_manager.ids;
- *vm_pd_addr = pd_addr;
- trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
- }
+ list_move_tail(&id->list, &adev->vm_manager.ids_lru);
+ atomic_long_set(&id->owner, (long)id);
+ vm->ids[ring->idx] = id;
+ *vm_id = id - adev->vm_manager.ids;
+ *vm_pd_addr = pd_addr;
+ trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
+
+error:
mutex_unlock(&adev->vm_manager.lock);
return r;
}
@@ -1467,6 +1468,7 @@
/* skip over VMID 0, since it is the system VM */
for (i = 1; i < adev->vm_manager.num_ids; ++i) {
amdgpu_vm_reset_id(adev, i);
+ amdgpu_sync_create(&adev->vm_manager.ids[i].active);
list_add_tail(&adev->vm_manager.ids[i].list,
&adev->vm_manager.ids_lru);
}
@@ -1488,7 +1490,8 @@
for (i = 0; i < AMDGPU_NUM_VM; ++i) {
struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
- fence_put(id->active);
+ fence_put(adev->vm_manager.ids[i].first);
+ amdgpu_sync_free(&adev->vm_manager.ids[i].active);
fence_put(id->flushed_updates);
}
}