drm/amdkfd: make reset wavefronts per process per device

This commit moves the reset wavefront flag to per process per device
data structure, so we can support multiple devices.

Signed-off-by: Ben Goz <ben.goz@amd.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 56b904f..8a1f999 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -173,7 +173,7 @@
 		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
 				pdd->dev->id, p->pasid);
 
-		if (p->reset_wavefronts)
+		if (pdd->reset_wavefronts)
 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
 
 		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
@@ -222,6 +222,7 @@
 					struct mm_struct *mm)
 {
 	struct kfd_process *p;
+	struct kfd_process_device *pdd = NULL;
 
 	/*
 	 * The kfd_process structure can not be free because the
@@ -240,6 +241,15 @@
 	/* In case our notifier is called before IOMMU notifier */
 	pqm_uninit(&p->pqm);
 
+	/* Iterate over all process device data structure and check
+	 * if we should reset all wavefronts */
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
+		if (pdd->reset_wavefronts) {
+			pr_warn("amdkfd: Resetting all wave fronts\n");
+			dbgdev_wave_reset_wavefronts(pdd->dev, p);
+			pdd->reset_wavefronts = false;
+		}
+
 	mutex_unlock(&p->mutex);
 
 	/*
@@ -305,8 +315,6 @@
 	if (kfd_init_apertures(process) != 0)
 		goto err_init_apretures;
 
-	process->reset_wavefronts = false;
-
 	return process;
 
 err_init_apretures:
@@ -348,6 +356,7 @@
 		INIT_LIST_HEAD(&pdd->qpd.queues_list);
 		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
 		pdd->qpd.dqm = dev->dqm;
+		pdd->reset_wavefronts = false;
 		list_add(&pdd->per_device_list, &p->per_device_data);
 	}
 
@@ -409,10 +418,12 @@
 		kfd_dbgmgr_destroy(dev->dbgmgr);
 
 	pqm_uninit(&p->pqm);
-	if (p->reset_wavefronts)
-		dbgdev_wave_reset_wavefronts(dev, p);
 
 	pdd = kfd_get_process_device_data(dev, p);
+	if (pdd->reset_wavefronts) {
+		dbgdev_wave_reset_wavefronts(pdd->dev, p);
+		pdd->reset_wavefronts = false;
+	}
 
 	/*
 	 * Just mark pdd as unbound, because we still need it to call