Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2015 Advanced Micro Devices, Inc. |
| 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice shall be included in |
| 12 | * all copies or substantial portions of the Software. |
| 13 | * |
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 | * OTHER DEALINGS IN THE SOFTWARE. |
| 21 | * |
| 22 | * |
| 23 | */ |
| 24 | #include <linux/kthread.h> |
| 25 | #include <linux/wait.h> |
| 26 | #include <linux/sched.h> |
| 27 | #include <drm/drmP.h> |
| 28 | #include "gpu_scheduler.h" |
| 29 | |
Chunming Zhou | 353da3c | 2015-09-07 16:06:53 +0800 | [diff] [blame] | 30 | #define CREATE_TRACE_POINTS |
| 31 | #include "gpu_sched_trace.h" |
| 32 | |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 33 | static struct amd_sched_job * |
| 34 | amd_sched_entity_pop_job(struct amd_sched_entity *entity); |
Christian König | 8807900 | 2015-08-24 14:29:40 +0200 | [diff] [blame] | 35 | static void amd_sched_wakeup(struct amd_gpu_scheduler *sched); |
| 36 | |
Chunming Zhou | f5617f9 | 2015-11-05 11:41:50 +0800 | [diff] [blame] | 37 | struct kmem_cache *sched_fence_slab; |
| 38 | atomic_t sched_fence_slab_ref = ATOMIC_INIT(0); |
| 39 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 40 | /* Initialize a given run queue struct */ |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 41 | static void amd_sched_rq_init(struct amd_sched_rq *rq) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 42 | { |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 43 | spin_lock_init(&rq->lock); |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 44 | INIT_LIST_HEAD(&rq->entities); |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 45 | rq->current_entity = NULL; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 46 | } |
| 47 | |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 48 | static void amd_sched_rq_add_entity(struct amd_sched_rq *rq, |
| 49 | struct amd_sched_entity *entity) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 50 | { |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 51 | spin_lock(&rq->lock); |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 52 | list_add_tail(&entity->list, &rq->entities); |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 53 | spin_unlock(&rq->lock); |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq, |
| 57 | struct amd_sched_entity *entity) |
| 58 | { |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 59 | spin_lock(&rq->lock); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 60 | list_del_init(&entity->list); |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 61 | if (rq->current_entity == entity) |
| 62 | rq->current_entity = NULL; |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 63 | spin_unlock(&rq->lock); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 64 | } |
| 65 | |
| 66 | /** |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 67 | * Select next job from a specified run queue with round robin policy. |
| 68 | * Return NULL if nothing available. |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 69 | */ |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 70 | static struct amd_sched_job * |
| 71 | amd_sched_rq_select_job(struct amd_sched_rq *rq) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 72 | { |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 73 | struct amd_sched_entity *entity; |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 74 | struct amd_sched_job *sched_job; |
Christian König | 4cd7f42c | 2015-08-05 18:18:52 +0200 | [diff] [blame] | 75 | |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 76 | spin_lock(&rq->lock); |
| 77 | |
| 78 | entity = rq->current_entity; |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 79 | if (entity) { |
| 80 | list_for_each_entry_continue(entity, &rq->entities, list) { |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 81 | sched_job = amd_sched_entity_pop_job(entity); |
| 82 | if (sched_job) { |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 83 | rq->current_entity = entity; |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 84 | spin_unlock(&rq->lock); |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 85 | return sched_job; |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 86 | } |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 87 | } |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 88 | } |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 89 | |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 90 | list_for_each_entry(entity, &rq->entities, list) { |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 91 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 92 | sched_job = amd_sched_entity_pop_job(entity); |
| 93 | if (sched_job) { |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 94 | rq->current_entity = entity; |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 95 | spin_unlock(&rq->lock); |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 96 | return sched_job; |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 97 | } |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 98 | |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 99 | if (entity == rq->current_entity) |
| 100 | break; |
| 101 | } |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 102 | |
Christian König | 2b184d8 | 2015-08-18 14:41:25 +0200 | [diff] [blame] | 103 | spin_unlock(&rq->lock); |
| 104 | |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 105 | return NULL; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 106 | } |
| 107 | |
| 108 | /** |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 109 | * Init a context entity used by scheduler when submit to HW ring. |
| 110 | * |
| 111 | * @sched The pointer to the scheduler |
Christian König | 91404fb | 2015-08-05 18:33:21 +0200 | [diff] [blame] | 112 | * @entity The pointer to a valid amd_sched_entity |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 113 | * @rq The run queue this entity belongs |
Christian König | 0e89d0c | 2015-08-04 16:58:36 +0200 | [diff] [blame] | 114 | * @kernel If this is an entity for the kernel |
Jammy Zhou | 1333f72 | 2015-07-30 16:36:58 +0800 | [diff] [blame] | 115 | * @jobs The max number of jobs in the job queue |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 116 | * |
| 117 | * return 0 if succeed. negative error code on failure |
| 118 | */ |
Christian König | 91404fb | 2015-08-05 18:33:21 +0200 | [diff] [blame] | 119 | int amd_sched_entity_init(struct amd_gpu_scheduler *sched, |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 120 | struct amd_sched_entity *entity, |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 121 | struct amd_sched_rq *rq, |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 122 | uint32_t jobs) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 123 | { |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 124 | int r; |
| 125 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 126 | if (!(sched && entity && rq)) |
| 127 | return -EINVAL; |
| 128 | |
Christian König | 91404fb | 2015-08-05 18:33:21 +0200 | [diff] [blame] | 129 | memset(entity, 0, sizeof(struct amd_sched_entity)); |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 130 | INIT_LIST_HEAD(&entity->list); |
| 131 | entity->rq = rq; |
| 132 | entity->sched = sched; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 133 | |
| 134 | spin_lock_init(&entity->queue_lock); |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 135 | r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL); |
| 136 | if (r) |
| 137 | return r; |
| 138 | |
Christian König | ce882e6 | 2015-08-19 15:00:55 +0200 | [diff] [blame] | 139 | atomic_set(&entity->fence_seq, 0); |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 140 | entity->fence_context = fence_context_alloc(1); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 141 | |
| 142 | /* Add the entity to the run queue */ |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 143 | amd_sched_rq_add_entity(rq, entity); |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 144 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 145 | return 0; |
| 146 | } |
| 147 | |
| 148 | /** |
| 149 | * Query if entity is initialized |
| 150 | * |
| 151 | * @sched Pointer to scheduler instance |
| 152 | * @entity The pointer to a valid scheduler entity |
| 153 | * |
| 154 | * return true if entity is initialized, false otherwise |
| 155 | */ |
Christian König | d54fdb9 | 2015-08-20 17:03:48 +0200 | [diff] [blame] | 156 | static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched, |
| 157 | struct amd_sched_entity *entity) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 158 | { |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 159 | return entity->sched == sched && |
| 160 | entity->rq != NULL; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 161 | } |
| 162 | |
Christian König | aef4852 | 2015-08-20 14:47:46 +0200 | [diff] [blame] | 163 | /** |
| 164 | * Check if entity is idle |
| 165 | * |
| 166 | * @entity The pointer to a valid scheduler entity |
| 167 | * |
| 168 | * Return true if entity don't has any unscheduled jobs. |
| 169 | */ |
| 170 | static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 171 | { |
Christian König | aef4852 | 2015-08-20 14:47:46 +0200 | [diff] [blame] | 172 | rmb(); |
| 173 | if (kfifo_is_empty(&entity->job_queue)) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 174 | return true; |
| 175 | |
| 176 | return false; |
| 177 | } |
| 178 | |
| 179 | /** |
| 180 | * Destroy a context entity |
| 181 | * |
| 182 | * @sched Pointer to scheduler instance |
| 183 | * @entity The pointer to a valid scheduler entity |
| 184 | * |
Christian König | 062c7fb | 2015-08-21 15:46:43 +0200 | [diff] [blame] | 185 | * Cleanup and free the allocated resources. |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 186 | */ |
Christian König | 062c7fb | 2015-08-21 15:46:43 +0200 | [diff] [blame] | 187 | void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, |
| 188 | struct amd_sched_entity *entity) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 189 | { |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 190 | struct amd_sched_rq *rq = entity->rq; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 191 | |
Christian König | d54fdb9 | 2015-08-20 17:03:48 +0200 | [diff] [blame] | 192 | if (!amd_sched_entity_is_initialized(sched, entity)) |
Christian König | 062c7fb | 2015-08-21 15:46:43 +0200 | [diff] [blame] | 193 | return; |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 194 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 195 | /** |
| 196 | * The client will not queue more IBs during this fini, consume existing |
| 197 | * queued IBs |
| 198 | */ |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 199 | wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity)); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 200 | |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 201 | amd_sched_rq_remove_entity(rq, entity); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 202 | kfifo_free(&entity->job_queue); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 203 | } |
| 204 | |
Christian König | e61235d | 2015-08-25 11:05:36 +0200 | [diff] [blame] | 205 | static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb) |
| 206 | { |
| 207 | struct amd_sched_entity *entity = |
| 208 | container_of(cb, struct amd_sched_entity, cb); |
| 209 | entity->dependency = NULL; |
| 210 | fence_put(f); |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 211 | amd_sched_wakeup(entity->sched); |
Christian König | e61235d | 2015-08-25 11:05:36 +0200 | [diff] [blame] | 212 | } |
| 213 | |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 214 | static struct amd_sched_job * |
| 215 | amd_sched_entity_pop_job(struct amd_sched_entity *entity) |
| 216 | { |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 217 | struct amd_gpu_scheduler *sched = entity->sched; |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 218 | struct amd_sched_job *sched_job; |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 219 | |
Christian König | e61235d | 2015-08-25 11:05:36 +0200 | [diff] [blame] | 220 | if (ACCESS_ONCE(entity->dependency)) |
| 221 | return NULL; |
| 222 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 223 | if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job))) |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 224 | return NULL; |
| 225 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 226 | while ((entity->dependency = sched->ops->dependency(sched_job))) { |
Christian König | e61235d | 2015-08-25 11:05:36 +0200 | [diff] [blame] | 227 | |
Christian König | fe537d0 | 2015-10-27 17:00:17 +0100 | [diff] [blame] | 228 | if (entity->dependency->context == entity->fence_context) { |
| 229 | /* We can ignore fences from ourself */ |
| 230 | fence_put(entity->dependency); |
| 231 | continue; |
| 232 | } |
| 233 | |
Christian König | e61235d | 2015-08-25 11:05:36 +0200 | [diff] [blame] | 234 | if (fence_add_callback(entity->dependency, &entity->cb, |
| 235 | amd_sched_entity_wakeup)) |
| 236 | fence_put(entity->dependency); |
| 237 | else |
| 238 | return NULL; |
| 239 | } |
| 240 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 241 | return sched_job; |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 242 | } |
| 243 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 244 | /** |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 245 | * Helper to submit a job to the job queue |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 246 | * |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 247 | * @sched_job The pointer to job required to submit |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 248 | * |
| 249 | * Returns true if we could submit the job. |
| 250 | */ |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 251 | static bool amd_sched_entity_in(struct amd_sched_job *sched_job) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 252 | { |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 253 | struct amd_sched_entity *entity = sched_job->s_entity; |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 254 | bool added, first = false; |
| 255 | |
| 256 | spin_lock(&entity->queue_lock); |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 257 | added = kfifo_in(&entity->job_queue, &sched_job, |
| 258 | sizeof(sched_job)) == sizeof(sched_job); |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 259 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 260 | if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job)) |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 261 | first = true; |
| 262 | |
| 263 | spin_unlock(&entity->queue_lock); |
| 264 | |
| 265 | /* first job wakes up scheduler */ |
| 266 | if (first) |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 267 | amd_sched_wakeup(sched_job->sched); |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 268 | |
| 269 | return added; |
| 270 | } |
| 271 | |
| 272 | /** |
| 273 | * Submit a job to the job queue |
| 274 | * |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 275 | * @sched_job The pointer to job required to submit |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 276 | * |
| 277 | * Returns 0 for success, negative error code otherwise. |
| 278 | */ |
Christian König | e284022 | 2015-11-05 19:49:48 +0100 | [diff] [blame] | 279 | void amd_sched_entity_push_job(struct amd_sched_job *sched_job) |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 280 | { |
| 281 | struct amd_sched_entity *entity = sched_job->s_entity; |
Christian König | 6c85927 | 2015-08-20 16:12:50 +0200 | [diff] [blame] | 282 | |
Christian König | 0f75aee | 2015-09-07 18:07:14 +0200 | [diff] [blame] | 283 | wait_event(entity->sched->job_scheduled, |
Chunming Zhou | c9f0fe5 | 2015-08-31 15:46:12 +0800 | [diff] [blame] | 284 | amd_sched_entity_in(sched_job)); |
Chunming Zhou | 353da3c | 2015-09-07 16:06:53 +0800 | [diff] [blame] | 285 | trace_amd_sched_job(sched_job); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 286 | } |
| 287 | |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 288 | /** |
| 289 | * Return ture if we can push more jobs to the hw. |
| 290 | */ |
| 291 | static bool amd_sched_ready(struct amd_gpu_scheduler *sched) |
| 292 | { |
| 293 | return atomic_read(&sched->hw_rq_count) < |
| 294 | sched->hw_submission_limit; |
| 295 | } |
| 296 | |
| 297 | /** |
Christian König | 8807900 | 2015-08-24 14:29:40 +0200 | [diff] [blame] | 298 | * Wake up the scheduler when it is ready |
| 299 | */ |
| 300 | static void amd_sched_wakeup(struct amd_gpu_scheduler *sched) |
| 301 | { |
| 302 | if (amd_sched_ready(sched)) |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 303 | wake_up_interruptible(&sched->wake_up_worker); |
Christian König | 8807900 | 2015-08-24 14:29:40 +0200 | [diff] [blame] | 304 | } |
| 305 | |
| 306 | /** |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 307 | * Select next to run |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 308 | */ |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 309 | static struct amd_sched_job * |
| 310 | amd_sched_select_job(struct amd_gpu_scheduler *sched) |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 311 | { |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 312 | struct amd_sched_job *sched_job; |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 313 | |
| 314 | if (!amd_sched_ready(sched)) |
| 315 | return NULL; |
| 316 | |
| 317 | /* Kernel run queue has higher priority than normal run queue*/ |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 318 | sched_job = amd_sched_rq_select_job(&sched->kernel_rq); |
| 319 | if (sched_job == NULL) |
| 320 | sched_job = amd_sched_rq_select_job(&sched->sched_rq); |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 321 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 322 | return sched_job; |
Christian König | e688b728 | 2015-08-20 17:01:01 +0200 | [diff] [blame] | 323 | } |
| 324 | |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 325 | static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) |
| 326 | { |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 327 | struct amd_sched_fence *s_fence = |
| 328 | container_of(cb, struct amd_sched_fence, cb); |
Christian König | 9b398fa | 2015-09-07 18:16:49 +0200 | [diff] [blame] | 329 | struct amd_gpu_scheduler *sched = s_fence->sched; |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 330 | unsigned long flags; |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 331 | |
Christian König | c746ba2 | 2015-08-19 16:12:15 +0200 | [diff] [blame] | 332 | atomic_dec(&sched->hw_rq_count); |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 333 | amd_sched_fence_signal(s_fence); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 334 | if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { |
Christian König | 424839a | 2015-11-02 16:25:10 +0100 | [diff] [blame] | 335 | cancel_delayed_work(&s_fence->dwork); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 336 | spin_lock_irqsave(&sched->fence_list_lock, flags); |
| 337 | list_del_init(&s_fence->list); |
| 338 | spin_unlock_irqrestore(&sched->fence_list_lock, flags); |
| 339 | } |
Chunming Zhou | 7034dec | 2015-11-11 14:56:00 +0800 | [diff] [blame] | 340 | trace_amd_sched_process_job(s_fence); |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 341 | fence_put(&s_fence->base); |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 342 | wake_up_interruptible(&sched->wake_up_worker); |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 343 | } |
| 344 | |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 345 | static void amd_sched_fence_work_func(struct work_struct *work) |
| 346 | { |
| 347 | struct amd_sched_fence *s_fence = |
| 348 | container_of(work, struct amd_sched_fence, dwork.work); |
| 349 | struct amd_gpu_scheduler *sched = s_fence->sched; |
| 350 | struct amd_sched_fence *entity, *tmp; |
| 351 | unsigned long flags; |
| 352 | |
| 353 | DRM_ERROR("[%s] scheduler is timeout!\n", sched->name); |
| 354 | |
| 355 | /* Clean all pending fences */ |
Junwei Zhang | 2fcef6e | 2015-10-13 11:14:23 +0800 | [diff] [blame] | 356 | spin_lock_irqsave(&sched->fence_list_lock, flags); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 357 | list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) { |
| 358 | DRM_ERROR(" fence no %d\n", entity->base.seqno); |
Junwei Zhang | 2fcef6e | 2015-10-13 11:14:23 +0800 | [diff] [blame] | 359 | cancel_delayed_work(&entity->dwork); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 360 | list_del_init(&entity->list); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 361 | fence_put(&entity->base); |
| 362 | } |
Junwei Zhang | 2fcef6e | 2015-10-13 11:14:23 +0800 | [diff] [blame] | 363 | spin_unlock_irqrestore(&sched->fence_list_lock, flags); |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 364 | } |
| 365 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 366 | static int amd_sched_main(void *param) |
| 367 | { |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 368 | struct sched_param sparam = {.sched_priority = 1}; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 369 | struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; |
Alex Deucher | 5134e99 | 2015-09-04 00:11:54 -0400 | [diff] [blame] | 370 | int r, count; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 371 | |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 372 | spin_lock_init(&sched->fence_list_lock); |
| 373 | INIT_LIST_HEAD(&sched->fence_list); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 374 | sched_setscheduler(current, SCHED_FIFO, &sparam); |
| 375 | |
| 376 | while (!kthread_should_stop()) { |
Christian König | 69bd5bf1 | 2015-08-26 11:31:23 +0200 | [diff] [blame] | 377 | struct amd_sched_entity *entity; |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 378 | struct amd_sched_fence *s_fence; |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 379 | struct amd_sched_job *sched_job; |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 380 | struct fence *fence; |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 381 | unsigned long flags; |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 382 | |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 383 | wait_event_interruptible(sched->wake_up_worker, |
Christian König | f85a6dd | 2015-08-19 17:37:52 +0200 | [diff] [blame] | 384 | kthread_should_stop() || |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 385 | (sched_job = amd_sched_select_job(sched))); |
Christian König | f85a6dd | 2015-08-19 17:37:52 +0200 | [diff] [blame] | 386 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 387 | if (!sched_job) |
Christian König | f85a6dd | 2015-08-19 17:37:52 +0200 | [diff] [blame] | 388 | continue; |
| 389 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 390 | entity = sched_job->s_entity; |
| 391 | s_fence = sched_job->s_fence; |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 392 | |
| 393 | if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { |
| 394 | INIT_DELAYED_WORK(&s_fence->dwork, amd_sched_fence_work_func); |
| 395 | schedule_delayed_work(&s_fence->dwork, sched->timeout); |
| 396 | spin_lock_irqsave(&sched->fence_list_lock, flags); |
| 397 | list_add_tail(&s_fence->list, &sched->fence_list); |
| 398 | spin_unlock_irqrestore(&sched->fence_list_lock, flags); |
| 399 | } |
| 400 | |
Christian König | b034b57 | 2015-08-20 17:08:25 +0200 | [diff] [blame] | 401 | atomic_inc(&sched->hw_rq_count); |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 402 | fence = sched->ops->run_job(sched_job); |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 403 | if (fence) { |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 404 | r = fence_add_callback(fence, &s_fence->cb, |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 405 | amd_sched_process_job); |
| 406 | if (r == -ENOENT) |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 407 | amd_sched_process_job(fence, &s_fence->cb); |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 408 | else if (r) |
| 409 | DRM_ERROR("fence add callback failed (%d)\n", r); |
| 410 | fence_put(fence); |
Christian König | 27439fc | 2015-09-02 12:03:06 +0200 | [diff] [blame] | 411 | } else { |
| 412 | DRM_ERROR("Failed to run job!\n"); |
Christian König | 258f3f9 | 2015-08-31 17:02:52 +0200 | [diff] [blame] | 413 | amd_sched_process_job(NULL, &s_fence->cb); |
Christian König | 6f0e54a | 2015-08-05 21:22:10 +0200 | [diff] [blame] | 414 | } |
Christian König | aef4852 | 2015-08-20 14:47:46 +0200 | [diff] [blame] | 415 | |
Junwei Zhang | 4c7eb91 | 2015-09-09 09:05:55 +0800 | [diff] [blame] | 416 | count = kfifo_out(&entity->job_queue, &sched_job, |
| 417 | sizeof(sched_job)); |
| 418 | WARN_ON(count != sizeof(sched_job)); |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 419 | wake_up(&sched->job_scheduled); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 420 | } |
| 421 | return 0; |
| 422 | } |
| 423 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 424 | /** |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 425 | * Init a gpu scheduler instance |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 426 | * |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 427 | * @sched The pointer to the scheduler |
Christian König | 69f7dd6 | 2015-08-20 17:24:40 +0200 | [diff] [blame] | 428 | * @ops The backend operations for this scheduler. |
Christian König | 69f7dd6 | 2015-08-20 17:24:40 +0200 | [diff] [blame] | 429 | * @hw_submissions Number of hw submissions to do. |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 430 | * @name Name used for debugging |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 431 | * |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 432 | * Return 0 on success, otherwise error code. |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 433 | */ |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 434 | int amd_sched_init(struct amd_gpu_scheduler *sched, |
| 435 | struct amd_sched_backend_ops *ops, |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 436 | unsigned hw_submission, long timeout, const char *name) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 437 | { |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 438 | sched->ops = ops; |
Chunming Zhou | 4cef926 | 2015-08-05 19:52:14 +0800 | [diff] [blame] | 439 | sched->hw_submission_limit = hw_submission; |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 440 | sched->name = name; |
Junwei Zhang | 2440ff2 | 2015-10-10 08:48:42 +0800 | [diff] [blame] | 441 | sched->timeout = timeout; |
Christian König | 432a4ff | 2015-08-12 11:46:04 +0200 | [diff] [blame] | 442 | amd_sched_rq_init(&sched->sched_rq); |
| 443 | amd_sched_rq_init(&sched->kernel_rq); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 444 | |
Christian König | c2b6bd7 | 2015-08-25 21:39:31 +0200 | [diff] [blame] | 445 | init_waitqueue_head(&sched->wake_up_worker); |
| 446 | init_waitqueue_head(&sched->job_scheduled); |
Christian König | c746ba2 | 2015-08-19 16:12:15 +0200 | [diff] [blame] | 447 | atomic_set(&sched->hw_rq_count, 0); |
Chunming Zhou | f5617f9 | 2015-11-05 11:41:50 +0800 | [diff] [blame] | 448 | if (atomic_inc_return(&sched_fence_slab_ref) == 1) { |
| 449 | sched_fence_slab = kmem_cache_create( |
| 450 | "amd_sched_fence", sizeof(struct amd_sched_fence), 0, |
| 451 | SLAB_HWCACHE_ALIGN, NULL); |
| 452 | if (!sched_fence_slab) |
| 453 | return -ENOMEM; |
| 454 | } |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 455 | |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 456 | /* Each scheduler will run on a seperate kernel thread */ |
Christian König | c14692f0 | 2015-08-21 15:18:47 +0200 | [diff] [blame] | 457 | sched->thread = kthread_run(amd_sched_main, sched, sched->name); |
Christian König | f495659 | 2015-08-20 16:59:38 +0200 | [diff] [blame] | 458 | if (IS_ERR(sched->thread)) { |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 459 | DRM_ERROR("Failed to create scheduler for %s.\n", name); |
| 460 | return PTR_ERR(sched->thread); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 461 | } |
| 462 | |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 463 | return 0; |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 464 | } |
| 465 | |
| 466 | /** |
| 467 | * Destroy a gpu scheduler |
| 468 | * |
| 469 | * @sched The pointer to the scheduler |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 470 | */ |
Christian König | 4f839a2 | 2015-09-08 20:22:31 +0200 | [diff] [blame] | 471 | void amd_sched_fini(struct amd_gpu_scheduler *sched) |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 472 | { |
Dave Airlie | 32544d0 | 2015-11-03 11:10:03 -0500 | [diff] [blame] | 473 | if (sched->thread) |
| 474 | kthread_stop(sched->thread); |
Chunming Zhou | f5617f9 | 2015-11-05 11:41:50 +0800 | [diff] [blame] | 475 | if (atomic_dec_and_test(&sched_fence_slab_ref)) |
| 476 | kmem_cache_destroy(sched_fence_slab); |
Jammy Zhou | a72ce6f | 2015-05-22 18:55:07 +0800 | [diff] [blame] | 477 | } |