Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 1 | /* sched.c - SPU scheduler. |
| 2 | * |
| 3 | * Copyright (C) IBM 2005 |
| 4 | * Author: Mark Nutter <mnutter@us.ibm.com> |
| 5 | * |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 6 | * 2006-03-31 NUMA domains added. |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License as published by |
| 10 | * the Free Software Foundation; either version 2, or (at your option) |
| 11 | * any later version. |
| 12 | * |
| 13 | * This program is distributed in the hope that it will be useful, |
| 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | * GNU General Public License for more details. |
| 17 | * |
| 18 | * You should have received a copy of the GNU General Public License |
| 19 | * along with this program; if not, write to the Free Software |
| 20 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 21 | */ |
| 22 | |
Arnd Bergmann | 3b3d22c | 2005-12-05 22:52:24 -0500 | [diff] [blame] | 23 | #undef DEBUG |
| 24 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 25 | #include <linux/module.h> |
| 26 | #include <linux/errno.h> |
| 27 | #include <linux/sched.h> |
| 28 | #include <linux/kernel.h> |
| 29 | #include <linux/mm.h> |
| 30 | #include <linux/completion.h> |
| 31 | #include <linux/vmalloc.h> |
| 32 | #include <linux/smp.h> |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 33 | #include <linux/stddef.h> |
| 34 | #include <linux/unistd.h> |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 35 | #include <linux/numa.h> |
| 36 | #include <linux/mutex.h> |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 37 | #include <linux/notifier.h> |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 38 | #include <linux/kthread.h> |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 39 | |
| 40 | #include <asm/io.h> |
| 41 | #include <asm/mmu_context.h> |
| 42 | #include <asm/spu.h> |
| 43 | #include <asm/spu_csa.h> |
Geoff Levand | a91942a | 2006-06-19 20:33:30 +0200 | [diff] [blame] | 44 | #include <asm/spu_priv1.h> |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 45 | #include "spufs.h" |
| 46 | |
Christoph Hellwig | 2eb1b12 | 2007-02-13 21:54:29 +0100 | [diff] [blame] | 47 | #define SPU_TIMESLICE (HZ) |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 48 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 49 | #define SPUSCHED_TICK (HZ / 100) |
| 50 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 51 | struct spu_prio_array { |
Christoph Hellwig | 72cb360 | 2007-02-13 21:54:28 +0100 | [diff] [blame] | 52 | DECLARE_BITMAP(bitmap, MAX_PRIO); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 53 | struct list_head runq[MAX_PRIO]; |
| 54 | spinlock_t runq_lock; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 55 | struct list_head active_list[MAX_NUMNODES]; |
| 56 | struct mutex active_mutex[MAX_NUMNODES]; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 57 | }; |
| 58 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 59 | static struct spu_prio_array *spu_prio; |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 60 | static struct task_struct *spusched_task; |
| 61 | static struct timer_list spusched_timer; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 62 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 63 | static inline int node_allowed(int node) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 64 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 65 | cpumask_t mask; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 66 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 67 | if (!nr_cpus_node(node)) |
| 68 | return 0; |
| 69 | mask = node_to_cpumask(node); |
| 70 | if (!cpus_intersects(mask, current->cpus_allowed)) |
| 71 | return 0; |
| 72 | return 1; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 73 | } |
| 74 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 75 | /** |
| 76 | * spu_add_to_active_list - add spu to active list |
| 77 | * @spu: spu to add to the active list |
| 78 | */ |
| 79 | static void spu_add_to_active_list(struct spu *spu) |
| 80 | { |
| 81 | mutex_lock(&spu_prio->active_mutex[spu->node]); |
| 82 | list_add_tail(&spu->list, &spu_prio->active_list[spu->node]); |
| 83 | mutex_unlock(&spu_prio->active_mutex[spu->node]); |
| 84 | } |
| 85 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 86 | static void __spu_remove_from_active_list(struct spu *spu) |
| 87 | { |
| 88 | list_del_init(&spu->list); |
| 89 | } |
| 90 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 91 | /** |
| 92 | * spu_remove_from_active_list - remove spu from active list |
| 93 | * @spu: spu to remove from the active list |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 94 | */ |
Christoph Hellwig | 678b2ff | 2007-02-13 21:54:25 +0100 | [diff] [blame] | 95 | static void spu_remove_from_active_list(struct spu *spu) |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 96 | { |
| 97 | int node = spu->node; |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 98 | |
| 99 | mutex_lock(&spu_prio->active_mutex[node]); |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 100 | __spu_remove_from_active_list(spu); |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 101 | mutex_unlock(&spu_prio->active_mutex[node]); |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 102 | } |
| 103 | |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 104 | static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier); |
| 105 | |
| 106 | static void spu_switch_notify(struct spu *spu, struct spu_context *ctx) |
| 107 | { |
| 108 | blocking_notifier_call_chain(&spu_switch_notifier, |
| 109 | ctx ? ctx->object_id : 0, spu); |
| 110 | } |
| 111 | |
| 112 | int spu_switch_event_register(struct notifier_block * n) |
| 113 | { |
| 114 | return blocking_notifier_chain_register(&spu_switch_notifier, n); |
| 115 | } |
| 116 | |
| 117 | int spu_switch_event_unregister(struct notifier_block * n) |
| 118 | { |
| 119 | return blocking_notifier_chain_unregister(&spu_switch_notifier, n); |
| 120 | } |
| 121 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 122 | /** |
| 123 | * spu_bind_context - bind spu context to physical spu |
| 124 | * @spu: physical spu to bind to |
| 125 | * @ctx: context to bind |
| 126 | */ |
| 127 | static void spu_bind_context(struct spu *spu, struct spu_context *ctx) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 128 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 129 | pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__, current->pid, |
| 130 | spu->number, spu->node); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 131 | spu->ctx = ctx; |
| 132 | spu->flags = 0; |
| 133 | ctx->spu = spu; |
| 134 | ctx->ops = &spu_hw_ops; |
| 135 | spu->pid = current->pid; |
Benjamin Herrenschmidt | 94b2a43 | 2007-03-10 00:05:37 +0100 | [diff] [blame] | 136 | spu_associate_mm(spu, ctx->owner); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 137 | spu->ibox_callback = spufs_ibox_callback; |
| 138 | spu->wbox_callback = spufs_wbox_callback; |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 139 | spu->stop_callback = spufs_stop_callback; |
Arnd Bergmann | a33a7d7 | 2006-03-23 00:00:11 +0100 | [diff] [blame] | 140 | spu->mfc_callback = spufs_mfc_callback; |
Arnd Bergmann | 9add11d | 2006-10-04 17:26:14 +0200 | [diff] [blame] | 141 | spu->dma_callback = spufs_dma_callback; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 142 | mb(); |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 143 | spu_unmap_mappings(ctx); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 144 | spu_restore(&ctx->csa, spu); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 145 | spu->timestamp = jiffies; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 146 | spu_cpu_affinity_set(spu, raw_smp_processor_id()); |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 147 | spu_switch_notify(spu, ctx); |
Christoph Hellwig | 81998ba | 2007-02-13 21:36:48 +0100 | [diff] [blame] | 148 | ctx->state = SPU_STATE_RUNNABLE; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 149 | } |
| 150 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 151 | /** |
| 152 | * spu_unbind_context - unbind spu context from physical spu |
| 153 | * @spu: physical spu to unbind from |
| 154 | * @ctx: context to unbind |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 155 | */ |
Christoph Hellwig | 678b2ff | 2007-02-13 21:54:25 +0100 | [diff] [blame] | 156 | static void spu_unbind_context(struct spu *spu, struct spu_context *ctx) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 157 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 158 | pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__, |
| 159 | spu->pid, spu->number, spu->node); |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 160 | |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 161 | spu_switch_notify(spu, NULL); |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 162 | spu_unmap_mappings(ctx); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 163 | spu_save(&ctx->csa, spu); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 164 | spu->timestamp = jiffies; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 165 | ctx->state = SPU_STATE_SAVED; |
| 166 | spu->ibox_callback = NULL; |
| 167 | spu->wbox_callback = NULL; |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 168 | spu->stop_callback = NULL; |
Arnd Bergmann | a33a7d7 | 2006-03-23 00:00:11 +0100 | [diff] [blame] | 169 | spu->mfc_callback = NULL; |
Arnd Bergmann | 9add11d | 2006-10-04 17:26:14 +0200 | [diff] [blame] | 170 | spu->dma_callback = NULL; |
Benjamin Herrenschmidt | 94b2a43 | 2007-03-10 00:05:37 +0100 | [diff] [blame] | 171 | spu_associate_mm(spu, NULL); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 172 | spu->pid = 0; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 173 | ctx->ops = &spu_backing_ops; |
| 174 | ctx->spu = NULL; |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 175 | spu->flags = 0; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 176 | spu->ctx = NULL; |
| 177 | } |
| 178 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 179 | /** |
| 180 | * spu_add_to_rq - add a context to the runqueue |
| 181 | * @ctx: context to add |
| 182 | */ |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 183 | static void __spu_add_to_rq(struct spu_context *ctx) |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 184 | { |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 185 | int prio = ctx->prio; |
| 186 | |
| 187 | list_add_tail(&ctx->rq, &spu_prio->runq[prio]); |
| 188 | set_bit(prio, spu_prio->bitmap); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 189 | } |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 190 | |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 191 | static void __spu_del_from_rq(struct spu_context *ctx) |
Christoph Hellwig | a475c2f | 2007-04-23 21:08:11 +0200 | [diff] [blame] | 192 | { |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 193 | int prio = ctx->prio; |
| 194 | |
Christoph Hellwig | a475c2f | 2007-04-23 21:08:11 +0200 | [diff] [blame] | 195 | if (!list_empty(&ctx->rq)) |
| 196 | list_del_init(&ctx->rq); |
| 197 | if (list_empty(&spu_prio->runq[prio])) |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 198 | clear_bit(prio, spu_prio->bitmap); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 199 | } |
| 200 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 201 | static void spu_prio_wait(struct spu_context *ctx) |
| 202 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 203 | DEFINE_WAIT(wait); |
| 204 | |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 205 | spin_lock(&spu_prio->runq_lock); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 206 | prepare_to_wait_exclusive(&ctx->stop_wq, &wait, TASK_INTERRUPTIBLE); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 207 | if (!signal_pending(current)) { |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 208 | __spu_add_to_rq(ctx); |
| 209 | spin_unlock(&spu_prio->runq_lock); |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 210 | mutex_unlock(&ctx->state_mutex); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 211 | schedule(); |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 212 | mutex_lock(&ctx->state_mutex); |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 213 | spin_lock(&spu_prio->runq_lock); |
| 214 | __spu_del_from_rq(ctx); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 215 | } |
Luke Browning | 4e0f4ed | 2007-04-23 21:08:13 +0200 | [diff] [blame] | 216 | spin_unlock(&spu_prio->runq_lock); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 217 | __set_current_state(TASK_RUNNING); |
| 218 | remove_wait_queue(&ctx->stop_wq, &wait); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 219 | } |
| 220 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 221 | static struct spu *spu_get_idle(struct spu_context *ctx) |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 222 | { |
| 223 | struct spu *spu = NULL; |
| 224 | int node = cpu_to_node(raw_smp_processor_id()); |
| 225 | int n; |
| 226 | |
| 227 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
| 228 | node = (node < MAX_NUMNODES) ? node : 0; |
| 229 | if (!node_allowed(node)) |
| 230 | continue; |
| 231 | spu = spu_alloc_node(node); |
| 232 | if (spu) |
| 233 | break; |
| 234 | } |
| 235 | return spu; |
| 236 | } |
| 237 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 238 | /** |
Christoph Hellwig | 52f04fc | 2007-02-13 21:54:27 +0100 | [diff] [blame] | 239 | * find_victim - find a lower priority context to preempt |
| 240 | * @ctx: canidate context for running |
| 241 | * |
| 242 | * Returns the freed physical spu to run the new context on. |
| 243 | */ |
| 244 | static struct spu *find_victim(struct spu_context *ctx) |
| 245 | { |
| 246 | struct spu_context *victim = NULL; |
| 247 | struct spu *spu; |
| 248 | int node, n; |
| 249 | |
| 250 | /* |
| 251 | * Look for a possible preemption candidate on the local node first. |
| 252 | * If there is no candidate look at the other nodes. This isn't |
| 253 | * exactly fair, but so far the whole spu schedule tries to keep |
| 254 | * a strong node affinity. We might want to fine-tune this in |
| 255 | * the future. |
| 256 | */ |
| 257 | restart: |
| 258 | node = cpu_to_node(raw_smp_processor_id()); |
| 259 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
| 260 | node = (node < MAX_NUMNODES) ? node : 0; |
| 261 | if (!node_allowed(node)) |
| 262 | continue; |
| 263 | |
| 264 | mutex_lock(&spu_prio->active_mutex[node]); |
| 265 | list_for_each_entry(spu, &spu_prio->active_list[node], list) { |
| 266 | struct spu_context *tmp = spu->ctx; |
| 267 | |
| 268 | if (tmp->rt_priority < ctx->rt_priority && |
| 269 | (!victim || tmp->rt_priority < victim->rt_priority)) |
| 270 | victim = spu->ctx; |
| 271 | } |
| 272 | mutex_unlock(&spu_prio->active_mutex[node]); |
| 273 | |
| 274 | if (victim) { |
| 275 | /* |
| 276 | * This nests ctx->state_mutex, but we always lock |
| 277 | * higher priority contexts before lower priority |
| 278 | * ones, so this is safe until we introduce |
| 279 | * priority inheritance schemes. |
| 280 | */ |
| 281 | if (!mutex_trylock(&victim->state_mutex)) { |
| 282 | victim = NULL; |
| 283 | goto restart; |
| 284 | } |
| 285 | |
| 286 | spu = victim->spu; |
| 287 | if (!spu) { |
| 288 | /* |
| 289 | * This race can happen because we've dropped |
| 290 | * the active list mutex. No a problem, just |
| 291 | * restart the search. |
| 292 | */ |
| 293 | mutex_unlock(&victim->state_mutex); |
| 294 | victim = NULL; |
| 295 | goto restart; |
| 296 | } |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 297 | spu_remove_from_active_list(spu); |
Christoph Hellwig | 52f04fc | 2007-02-13 21:54:27 +0100 | [diff] [blame] | 298 | spu_unbind_context(spu, victim); |
| 299 | mutex_unlock(&victim->state_mutex); |
Christoph Hellwig | e097b51 | 2007-04-23 21:08:09 +0200 | [diff] [blame] | 300 | /* |
| 301 | * We need to break out of the wait loop in spu_run |
| 302 | * manually to ensure this context gets put on the |
| 303 | * runqueue again ASAP. |
| 304 | */ |
| 305 | wake_up(&victim->stop_wq); |
Christoph Hellwig | 52f04fc | 2007-02-13 21:54:27 +0100 | [diff] [blame] | 306 | return spu; |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | return NULL; |
| 311 | } |
| 312 | |
| 313 | /** |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 314 | * spu_activate - find a free spu for a context and execute it |
| 315 | * @ctx: spu context to schedule |
| 316 | * @flags: flags (currently ignored) |
| 317 | * |
Christoph Hellwig | 0887309 | 2007-04-23 21:08:06 +0200 | [diff] [blame] | 318 | * Tries to find a free spu to run @ctx. If no free spu is available |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 319 | * add the context to the runqueue so it gets woken up once an spu |
| 320 | * is available. |
| 321 | */ |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame] | 322 | int spu_activate(struct spu_context *ctx, unsigned long flags) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 323 | { |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 324 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 325 | if (ctx->spu) |
| 326 | return 0; |
| 327 | |
| 328 | do { |
| 329 | struct spu *spu; |
| 330 | |
| 331 | spu = spu_get_idle(ctx); |
Christoph Hellwig | 52f04fc | 2007-02-13 21:54:27 +0100 | [diff] [blame] | 332 | /* |
| 333 | * If this is a realtime thread we try to get it running by |
| 334 | * preempting a lower priority thread. |
| 335 | */ |
| 336 | if (!spu && ctx->rt_priority) |
| 337 | spu = find_victim(ctx); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 338 | if (spu) { |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 339 | spu_bind_context(spu, ctx); |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 340 | spu_add_to_active_list(spu); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 341 | return 0; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 342 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 343 | |
Christoph Hellwig | 50b520d | 2007-03-10 00:05:36 +0100 | [diff] [blame] | 344 | spu_prio_wait(ctx); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 345 | } while (!signal_pending(current)); |
| 346 | |
| 347 | return -ERESTARTSYS; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 348 | } |
| 349 | |
Christoph Hellwig | 678b2ff | 2007-02-13 21:54:25 +0100 | [diff] [blame] | 350 | /** |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 351 | * grab_runnable_context - try to find a runnable context |
| 352 | * |
| 353 | * Remove the highest priority context on the runqueue and return it |
| 354 | * to the caller. Returns %NULL if no runnable context was found. |
| 355 | */ |
| 356 | static struct spu_context *grab_runnable_context(int prio) |
| 357 | { |
| 358 | struct spu_context *ctx = NULL; |
| 359 | int best; |
| 360 | |
| 361 | spin_lock(&spu_prio->runq_lock); |
| 362 | best = sched_find_first_bit(spu_prio->bitmap); |
| 363 | if (best < prio) { |
| 364 | struct list_head *rq = &spu_prio->runq[best]; |
| 365 | |
| 366 | BUG_ON(list_empty(rq)); |
| 367 | |
| 368 | ctx = list_entry(rq->next, struct spu_context, rq); |
| 369 | __spu_del_from_rq(ctx); |
| 370 | } |
| 371 | spin_unlock(&spu_prio->runq_lock); |
| 372 | |
| 373 | return ctx; |
| 374 | } |
| 375 | |
| 376 | static int __spu_deactivate(struct spu_context *ctx, int force, int max_prio) |
| 377 | { |
| 378 | struct spu *spu = ctx->spu; |
| 379 | struct spu_context *new = NULL; |
| 380 | |
| 381 | if (spu) { |
| 382 | new = grab_runnable_context(max_prio); |
| 383 | if (new || force) { |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 384 | spu_remove_from_active_list(spu); |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 385 | spu_unbind_context(spu, ctx); |
| 386 | spu_free(spu); |
| 387 | if (new) |
| 388 | wake_up(&new->stop_wq); |
| 389 | } |
| 390 | |
| 391 | } |
| 392 | |
| 393 | return new != NULL; |
| 394 | } |
| 395 | |
| 396 | /** |
Christoph Hellwig | 678b2ff | 2007-02-13 21:54:25 +0100 | [diff] [blame] | 397 | * spu_deactivate - unbind a context from it's physical spu |
| 398 | * @ctx: spu context to unbind |
| 399 | * |
| 400 | * Unbind @ctx from the physical spu it is running on and schedule |
| 401 | * the highest priority context to run on the freed physical spu. |
| 402 | */ |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 403 | void spu_deactivate(struct spu_context *ctx) |
| 404 | { |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 405 | __spu_deactivate(ctx, 1, MAX_PRIO); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 406 | } |
| 407 | |
Christoph Hellwig | ae7b4c5 | 2007-02-13 21:54:26 +0100 | [diff] [blame] | 408 | /** |
| 409 | * spu_yield - yield a physical spu if others are waiting |
| 410 | * @ctx: spu context to yield |
| 411 | * |
| 412 | * Check if there is a higher priority context waiting and if yes |
| 413 | * unbind @ctx from the physical spu and schedule the highest |
| 414 | * priority context to run on the freed physical spu instead. |
| 415 | */ |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 416 | void spu_yield(struct spu_context *ctx) |
| 417 | { |
Christoph Hellwig | e5c0b9e | 2007-06-05 11:25:59 +1000 | [diff] [blame] | 418 | if (!(ctx->flags & SPU_CREATE_NOSCHED)) { |
| 419 | mutex_lock(&ctx->state_mutex); |
| 420 | __spu_deactivate(ctx, 0, MAX_PRIO); |
| 421 | mutex_unlock(&ctx->state_mutex); |
| 422 | } |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 423 | } |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 424 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 425 | static void spusched_tick(struct spu_context *ctx) |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 426 | { |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 427 | if (ctx->policy != SCHED_RR || --ctx->time_slice) |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 428 | return; |
| 429 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 430 | /* |
| 431 | * Unfortunately active_mutex ranks outside of state_mutex, so |
| 432 | * we have to trylock here. If we fail give the context another |
| 433 | * tick and try again. |
| 434 | */ |
| 435 | if (mutex_trylock(&ctx->state_mutex)) { |
| 436 | struct spu_context *new = grab_runnable_context(ctx->prio + 1); |
| 437 | if (new) { |
| 438 | struct spu *spu = ctx->spu; |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 439 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 440 | __spu_remove_from_active_list(spu); |
| 441 | spu_unbind_context(spu, ctx); |
| 442 | spu_free(spu); |
| 443 | wake_up(&new->stop_wq); |
| 444 | /* |
| 445 | * We need to break out of the wait loop in |
| 446 | * spu_run manually to ensure this context |
| 447 | * gets put on the runqueue again ASAP. |
| 448 | */ |
| 449 | wake_up(&ctx->stop_wq); |
| 450 | } |
| 451 | ctx->time_slice = SPU_DEF_TIMESLICE; |
| 452 | mutex_unlock(&ctx->state_mutex); |
Christoph Hellwig | bb5db29 | 2007-06-04 23:26:51 +1000 | [diff] [blame] | 453 | } else { |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 454 | ctx->time_slice++; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 455 | } |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 456 | } |
| 457 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 458 | static void spusched_wake(unsigned long data) |
| 459 | { |
| 460 | mod_timer(&spusched_timer, jiffies + SPUSCHED_TICK); |
| 461 | wake_up_process(spusched_task); |
| 462 | } |
| 463 | |
| 464 | static int spusched_thread(void *unused) |
| 465 | { |
| 466 | struct spu *spu, *next; |
| 467 | int node; |
| 468 | |
| 469 | setup_timer(&spusched_timer, spusched_wake, 0); |
| 470 | __mod_timer(&spusched_timer, jiffies + SPUSCHED_TICK); |
| 471 | |
| 472 | while (!kthread_should_stop()) { |
| 473 | set_current_state(TASK_INTERRUPTIBLE); |
| 474 | schedule(); |
| 475 | for (node = 0; node < MAX_NUMNODES; node++) { |
| 476 | mutex_lock(&spu_prio->active_mutex[node]); |
| 477 | list_for_each_entry_safe(spu, next, |
| 478 | &spu_prio->active_list[node], |
| 479 | list) |
| 480 | spusched_tick(spu->ctx); |
| 481 | mutex_unlock(&spu_prio->active_mutex[node]); |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | del_timer_sync(&spusched_timer); |
| 486 | return 0; |
| 487 | } |
| 488 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 489 | int __init spu_sched_init(void) |
| 490 | { |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 491 | int i; |
| 492 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 493 | spu_prio = kzalloc(sizeof(struct spu_prio_array), GFP_KERNEL); |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 494 | if (!spu_prio) |
| 495 | return -ENOMEM; |
| 496 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 497 | for (i = 0; i < MAX_PRIO; i++) { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 498 | INIT_LIST_HEAD(&spu_prio->runq[i]); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 499 | __clear_bit(i, spu_prio->bitmap); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 500 | } |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 501 | __set_bit(MAX_PRIO, spu_prio->bitmap); |
| 502 | for (i = 0; i < MAX_NUMNODES; i++) { |
| 503 | mutex_init(&spu_prio->active_mutex[i]); |
| 504 | INIT_LIST_HEAD(&spu_prio->active_list[i]); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 505 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 506 | spin_lock_init(&spu_prio->runq_lock); |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 507 | |
| 508 | spusched_task = kthread_run(spusched_thread, NULL, "spusched"); |
| 509 | if (IS_ERR(spusched_task)) { |
| 510 | kfree(spu_prio); |
| 511 | return PTR_ERR(spusched_task); |
| 512 | } |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 513 | return 0; |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 514 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 515 | } |
| 516 | |
| 517 | void __exit spu_sched_exit(void) |
| 518 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 519 | struct spu *spu, *tmp; |
| 520 | int node; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 521 | |
Christoph Hellwig | 3790180 | 2007-06-29 10:57:51 +1000 | [diff] [blame^] | 522 | kthread_stop(spusched_task); |
| 523 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 524 | for (node = 0; node < MAX_NUMNODES; node++) { |
| 525 | mutex_lock(&spu_prio->active_mutex[node]); |
| 526 | list_for_each_entry_safe(spu, tmp, &spu_prio->active_list[node], |
| 527 | list) { |
| 528 | list_del_init(&spu->list); |
| 529 | spu_free(spu); |
| 530 | } |
| 531 | mutex_unlock(&spu_prio->active_mutex[node]); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 532 | } |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 533 | kfree(spu_prio); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 534 | } |