Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 1 | /* sched.c - SPU scheduler. |
| 2 | * |
| 3 | * Copyright (C) IBM 2005 |
| 4 | * Author: Mark Nutter <mnutter@us.ibm.com> |
| 5 | * |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 6 | * 2006-03-31 NUMA domains added. |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License as published by |
| 10 | * the Free Software Foundation; either version 2, or (at your option) |
| 11 | * any later version. |
| 12 | * |
| 13 | * This program is distributed in the hope that it will be useful, |
| 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | * GNU General Public License for more details. |
| 17 | * |
| 18 | * You should have received a copy of the GNU General Public License |
| 19 | * along with this program; if not, write to the Free Software |
| 20 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 21 | */ |
| 22 | |
Arnd Bergmann | 3b3d22c | 2005-12-05 22:52:24 -0500 | [diff] [blame] | 23 | #undef DEBUG |
| 24 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 25 | #include <linux/module.h> |
| 26 | #include <linux/errno.h> |
| 27 | #include <linux/sched.h> |
| 28 | #include <linux/kernel.h> |
| 29 | #include <linux/mm.h> |
| 30 | #include <linux/completion.h> |
| 31 | #include <linux/vmalloc.h> |
| 32 | #include <linux/smp.h> |
| 33 | #include <linux/smp_lock.h> |
| 34 | #include <linux/stddef.h> |
| 35 | #include <linux/unistd.h> |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 36 | #include <linux/numa.h> |
| 37 | #include <linux/mutex.h> |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 38 | #include <linux/notifier.h> |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 39 | |
| 40 | #include <asm/io.h> |
| 41 | #include <asm/mmu_context.h> |
| 42 | #include <asm/spu.h> |
| 43 | #include <asm/spu_csa.h> |
Geoff Levand | a91942a | 2006-06-19 20:33:30 +0200 | [diff] [blame] | 44 | #include <asm/spu_priv1.h> |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 45 | #include "spufs.h" |
| 46 | |
Arnd Bergmann | 7945a4a | 2005-12-09 19:04:16 +0100 | [diff] [blame] | 47 | #define SPU_MIN_TIMESLICE (100 * HZ / 1000) |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 48 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 49 | #define SPU_BITMAP_SIZE (((MAX_PRIO+BITS_PER_LONG)/BITS_PER_LONG)+1) |
| 50 | struct spu_prio_array { |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 51 | unsigned long bitmap[SPU_BITMAP_SIZE]; |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 52 | struct list_head runq[MAX_PRIO]; |
| 53 | spinlock_t runq_lock; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 54 | struct list_head active_list[MAX_NUMNODES]; |
| 55 | struct mutex active_mutex[MAX_NUMNODES]; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 56 | }; |
| 57 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 58 | static struct spu_prio_array *spu_prio; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 59 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 60 | static inline int node_allowed(int node) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 61 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 62 | cpumask_t mask; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 63 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 64 | if (!nr_cpus_node(node)) |
| 65 | return 0; |
| 66 | mask = node_to_cpumask(node); |
| 67 | if (!cpus_intersects(mask, current->cpus_allowed)) |
| 68 | return 0; |
| 69 | return 1; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 70 | } |
| 71 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 72 | /** |
| 73 | * spu_add_to_active_list - add spu to active list |
| 74 | * @spu: spu to add to the active list |
| 75 | */ |
| 76 | static void spu_add_to_active_list(struct spu *spu) |
| 77 | { |
| 78 | mutex_lock(&spu_prio->active_mutex[spu->node]); |
| 79 | list_add_tail(&spu->list, &spu_prio->active_list[spu->node]); |
| 80 | mutex_unlock(&spu_prio->active_mutex[spu->node]); |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * spu_remove_from_active_list - remove spu from active list |
| 85 | * @spu: spu to remove from the active list |
| 86 | * |
| 87 | * This function removes an spu from the active list. If the spu was |
| 88 | * found on the active list the function returns 1, else it doesn't do |
| 89 | * anything and returns 0. |
| 90 | */ |
| 91 | static int spu_remove_from_active_list(struct spu *spu) |
| 92 | { |
| 93 | int node = spu->node; |
| 94 | struct spu *tmp; |
| 95 | int rc = 0; |
| 96 | |
| 97 | mutex_lock(&spu_prio->active_mutex[node]); |
| 98 | list_for_each_entry(tmp, &spu_prio->active_list[node], list) { |
| 99 | if (tmp == spu) { |
| 100 | list_del_init(&spu->list); |
| 101 | rc = 1; |
| 102 | break; |
| 103 | } |
| 104 | } |
| 105 | mutex_unlock(&spu_prio->active_mutex[node]); |
| 106 | return rc; |
| 107 | } |
| 108 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 109 | static inline void mm_needs_global_tlbie(struct mm_struct *mm) |
| 110 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 111 | int nr = (NR_CPUS > 1) ? NR_CPUS : NR_CPUS + 1; |
| 112 | |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 113 | /* Global TLBIE broadcast required with SPEs. */ |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 114 | __cpus_setall(&mm->cpu_vm_mask, nr); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 115 | } |
| 116 | |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 117 | static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier); |
| 118 | |
| 119 | static void spu_switch_notify(struct spu *spu, struct spu_context *ctx) |
| 120 | { |
| 121 | blocking_notifier_call_chain(&spu_switch_notifier, |
| 122 | ctx ? ctx->object_id : 0, spu); |
| 123 | } |
| 124 | |
| 125 | int spu_switch_event_register(struct notifier_block * n) |
| 126 | { |
| 127 | return blocking_notifier_chain_register(&spu_switch_notifier, n); |
| 128 | } |
| 129 | |
| 130 | int spu_switch_event_unregister(struct notifier_block * n) |
| 131 | { |
| 132 | return blocking_notifier_chain_unregister(&spu_switch_notifier, n); |
| 133 | } |
| 134 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 135 | /** |
| 136 | * spu_bind_context - bind spu context to physical spu |
| 137 | * @spu: physical spu to bind to |
| 138 | * @ctx: context to bind |
| 139 | */ |
| 140 | static void spu_bind_context(struct spu *spu, struct spu_context *ctx) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 141 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 142 | pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__, current->pid, |
| 143 | spu->number, spu->node); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 144 | spu->ctx = ctx; |
| 145 | spu->flags = 0; |
| 146 | ctx->spu = spu; |
| 147 | ctx->ops = &spu_hw_ops; |
| 148 | spu->pid = current->pid; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 149 | spu->mm = ctx->owner; |
| 150 | mm_needs_global_tlbie(spu->mm); |
| 151 | spu->ibox_callback = spufs_ibox_callback; |
| 152 | spu->wbox_callback = spufs_wbox_callback; |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 153 | spu->stop_callback = spufs_stop_callback; |
Arnd Bergmann | a33a7d7 | 2006-03-23 00:00:11 +0100 | [diff] [blame] | 154 | spu->mfc_callback = spufs_mfc_callback; |
Arnd Bergmann | 9add11d | 2006-10-04 17:26:14 +0200 | [diff] [blame] | 155 | spu->dma_callback = spufs_dma_callback; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 156 | mb(); |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 157 | spu_unmap_mappings(ctx); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 158 | spu_restore(&ctx->csa, spu); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 159 | spu->timestamp = jiffies; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 160 | spu_cpu_affinity_set(spu, raw_smp_processor_id()); |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 161 | spu_switch_notify(spu, ctx); |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 162 | spu_add_to_active_list(spu); |
Christoph Hellwig | 81998ba | 2007-02-13 21:36:48 +0100 | [diff] [blame] | 163 | ctx->state = SPU_STATE_RUNNABLE; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 164 | } |
| 165 | |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 166 | /** |
| 167 | * spu_unbind_context - unbind spu context from physical spu |
| 168 | * @spu: physical spu to unbind from |
| 169 | * @ctx: context to unbind |
| 170 | * |
| 171 | * If the spu was on the active list the function returns 1, else 0. |
| 172 | */ |
| 173 | static int spu_unbind_context(struct spu *spu, struct spu_context *ctx) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 174 | { |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 175 | int was_active = spu_remove_from_active_list(spu); |
| 176 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 177 | pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__, |
| 178 | spu->pid, spu->number, spu->node); |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 179 | |
Arnd Bergmann | 8676727 | 2006-10-04 17:26:21 +0200 | [diff] [blame] | 180 | spu_switch_notify(spu, NULL); |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 181 | spu_unmap_mappings(ctx); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 182 | spu_save(&ctx->csa, spu); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 183 | spu->timestamp = jiffies; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 184 | ctx->state = SPU_STATE_SAVED; |
| 185 | spu->ibox_callback = NULL; |
| 186 | spu->wbox_callback = NULL; |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 187 | spu->stop_callback = NULL; |
Arnd Bergmann | a33a7d7 | 2006-03-23 00:00:11 +0100 | [diff] [blame] | 188 | spu->mfc_callback = NULL; |
Arnd Bergmann | 9add11d | 2006-10-04 17:26:14 +0200 | [diff] [blame] | 189 | spu->dma_callback = NULL; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 190 | spu->mm = NULL; |
| 191 | spu->pid = 0; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 192 | ctx->ops = &spu_backing_ops; |
| 193 | ctx->spu = NULL; |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 194 | spu->flags = 0; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 195 | spu->ctx = NULL; |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 196 | |
| 197 | return was_active; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 198 | } |
| 199 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 200 | /** |
| 201 | * spu_add_to_rq - add a context to the runqueue |
| 202 | * @ctx: context to add |
| 203 | */ |
| 204 | static void spu_add_to_rq(struct spu_context *ctx) |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 205 | { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 206 | spin_lock(&spu_prio->runq_lock); |
| 207 | list_add_tail(&ctx->rq, &spu_prio->runq[ctx->prio]); |
| 208 | set_bit(ctx->prio, spu_prio->bitmap); |
| 209 | spin_unlock(&spu_prio->runq_lock); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 210 | } |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 211 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 212 | /** |
| 213 | * spu_del_from_rq - remove a context from the runqueue |
| 214 | * @ctx: context to remove |
| 215 | */ |
| 216 | static void spu_del_from_rq(struct spu_context *ctx) |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 217 | { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 218 | spin_lock(&spu_prio->runq_lock); |
| 219 | list_del_init(&ctx->rq); |
| 220 | if (list_empty(&spu_prio->runq[ctx->prio])) |
| 221 | clear_bit(ctx->prio, spu_prio->bitmap); |
| 222 | spin_unlock(&spu_prio->runq_lock); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 223 | } |
| 224 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 225 | /** |
| 226 | * spu_grab_context - remove one context from the runqueue |
| 227 | * @prio: priority of the context to be removed |
| 228 | * |
| 229 | * This function removes one context from the runqueue for priority @prio. |
| 230 | * If there is more than one context with the given priority the first |
| 231 | * task on the runqueue will be taken. |
| 232 | * |
| 233 | * Returns the spu_context it just removed. |
| 234 | * |
| 235 | * Must be called with spu_prio->runq_lock held. |
| 236 | */ |
| 237 | static struct spu_context *spu_grab_context(int prio) |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 238 | { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 239 | struct list_head *rq = &spu_prio->runq[prio]; |
| 240 | |
| 241 | if (list_empty(rq)) |
| 242 | return NULL; |
| 243 | return list_entry(rq->next, struct spu_context, rq); |
| 244 | } |
| 245 | |
| 246 | static void spu_prio_wait(struct spu_context *ctx) |
| 247 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 248 | DEFINE_WAIT(wait); |
| 249 | |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame^] | 250 | set_bit(SPU_SCHED_WAKE, &ctx->sched_flags); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 251 | prepare_to_wait_exclusive(&ctx->stop_wq, &wait, TASK_INTERRUPTIBLE); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 252 | if (!signal_pending(current)) { |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 253 | mutex_unlock(&ctx->state_mutex); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 254 | schedule(); |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 255 | mutex_lock(&ctx->state_mutex); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 256 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 257 | __set_current_state(TASK_RUNNING); |
| 258 | remove_wait_queue(&ctx->stop_wq, &wait); |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame^] | 259 | clear_bit(SPU_SCHED_WAKE, &ctx->sched_flags); |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 260 | } |
| 261 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 262 | /** |
| 263 | * spu_reschedule - try to find a runnable context for a spu |
| 264 | * @spu: spu available |
| 265 | * |
| 266 | * This function is called whenever a spu becomes idle. It looks for the |
| 267 | * most suitable runnable spu context and schedules it for execution. |
| 268 | */ |
| 269 | static void spu_reschedule(struct spu *spu) |
Arnd Bergmann | 2a911f0 | 2005-12-05 22:52:26 -0500 | [diff] [blame] | 270 | { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 271 | int best; |
| 272 | |
| 273 | spu_free(spu); |
| 274 | |
| 275 | spin_lock(&spu_prio->runq_lock); |
| 276 | best = sched_find_first_bit(spu_prio->bitmap); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 277 | if (best < MAX_PRIO) { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 278 | struct spu_context *ctx = spu_grab_context(best); |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame^] | 279 | if (ctx && test_bit(SPU_SCHED_WAKE, &ctx->sched_flags)) |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 280 | wake_up(&ctx->stop_wq); |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 281 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 282 | spin_unlock(&spu_prio->runq_lock); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 283 | } |
| 284 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 285 | static struct spu *spu_get_idle(struct spu_context *ctx) |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 286 | { |
| 287 | struct spu *spu = NULL; |
| 288 | int node = cpu_to_node(raw_smp_processor_id()); |
| 289 | int n; |
| 290 | |
| 291 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
| 292 | node = (node < MAX_NUMNODES) ? node : 0; |
| 293 | if (!node_allowed(node)) |
| 294 | continue; |
| 295 | spu = spu_alloc_node(node); |
| 296 | if (spu) |
| 297 | break; |
| 298 | } |
| 299 | return spu; |
| 300 | } |
| 301 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 302 | /* The three externally callable interfaces |
| 303 | * for the scheduler begin here. |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 304 | * |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 305 | * spu_activate - bind a context to SPU, waiting as needed. |
| 306 | * spu_deactivate - unbind a context from its SPU. |
| 307 | * spu_yield - yield an SPU if others are waiting. |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 308 | */ |
| 309 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 310 | /** |
| 311 | * spu_activate - find a free spu for a context and execute it |
| 312 | * @ctx: spu context to schedule |
| 313 | * @flags: flags (currently ignored) |
| 314 | * |
| 315 | * Tries to find a free spu to run @ctx. If no free spu is availble |
| 316 | * add the context to the runqueue so it gets woken up once an spu |
| 317 | * is available. |
| 318 | */ |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame^] | 319 | int spu_activate(struct spu_context *ctx, unsigned long flags) |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 320 | { |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 321 | |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 322 | if (ctx->spu) |
| 323 | return 0; |
| 324 | |
| 325 | do { |
| 326 | struct spu *spu; |
| 327 | |
| 328 | spu = spu_get_idle(ctx); |
| 329 | if (spu) { |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 330 | spu_bind_context(spu, ctx); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 331 | return 0; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 332 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 333 | |
| 334 | spu_add_to_rq(ctx); |
Christoph Hellwig | 26bec67 | 2007-02-13 21:54:24 +0100 | [diff] [blame^] | 335 | if (!(flags & SPU_ACTIVATE_NOWAKE)) |
| 336 | spu_prio_wait(ctx); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 337 | spu_del_from_rq(ctx); |
| 338 | } while (!signal_pending(current)); |
| 339 | |
| 340 | return -ERESTARTSYS; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 341 | } |
| 342 | |
| 343 | void spu_deactivate(struct spu_context *ctx) |
| 344 | { |
| 345 | struct spu *spu; |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 346 | int was_active; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 347 | |
| 348 | spu = ctx->spu; |
| 349 | if (!spu) |
| 350 | return; |
Christoph Hellwig | 202557d | 2007-02-13 21:36:49 +0100 | [diff] [blame] | 351 | was_active = spu_unbind_context(spu, ctx); |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 352 | if (was_active) |
| 353 | spu_reschedule(spu); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 354 | } |
| 355 | |
| 356 | void spu_yield(struct spu_context *ctx) |
| 357 | { |
| 358 | struct spu *spu; |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 359 | int need_yield = 0; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 360 | |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 361 | if (mutex_trylock(&ctx->state_mutex)) { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 362 | if ((spu = ctx->spu) != NULL) { |
| 363 | int best = sched_find_first_bit(spu_prio->bitmap); |
| 364 | if (best < MAX_PRIO) { |
| 365 | pr_debug("%s: yielding SPU %d NODE %d\n", |
| 366 | __FUNCTION__, spu->number, spu->node); |
| 367 | spu_deactivate(ctx); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 368 | need_yield = 1; |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 369 | } |
| 370 | } |
Christoph Hellwig | 650f8b0 | 2007-02-13 21:36:50 +0100 | [diff] [blame] | 371 | mutex_unlock(&ctx->state_mutex); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 372 | } |
Arnd Bergmann | 5110459 | 2005-12-05 22:52:25 -0500 | [diff] [blame] | 373 | if (unlikely(need_yield)) |
| 374 | yield(); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 375 | } |
| 376 | |
| 377 | int __init spu_sched_init(void) |
| 378 | { |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 379 | int i; |
| 380 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 381 | spu_prio = kzalloc(sizeof(struct spu_prio_array), GFP_KERNEL); |
| 382 | if (!spu_prio) { |
| 383 | printk(KERN_WARNING "%s: Unable to allocate priority queue.\n", |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 384 | __FUNCTION__); |
| 385 | return 1; |
| 386 | } |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 387 | for (i = 0; i < MAX_PRIO; i++) { |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 388 | INIT_LIST_HEAD(&spu_prio->runq[i]); |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 389 | __clear_bit(i, spu_prio->bitmap); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 390 | } |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 391 | __set_bit(MAX_PRIO, spu_prio->bitmap); |
| 392 | for (i = 0; i < MAX_NUMNODES; i++) { |
| 393 | mutex_init(&spu_prio->active_mutex[i]); |
| 394 | INIT_LIST_HEAD(&spu_prio->active_list[i]); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 395 | } |
Christoph Hellwig | 079cdb6 | 2007-02-13 21:54:23 +0100 | [diff] [blame] | 396 | spin_lock_init(&spu_prio->runq_lock); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 397 | return 0; |
| 398 | } |
| 399 | |
| 400 | void __exit spu_sched_exit(void) |
| 401 | { |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 402 | struct spu *spu, *tmp; |
| 403 | int node; |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 404 | |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 405 | for (node = 0; node < MAX_NUMNODES; node++) { |
| 406 | mutex_lock(&spu_prio->active_mutex[node]); |
| 407 | list_for_each_entry_safe(spu, tmp, &spu_prio->active_list[node], |
| 408 | list) { |
| 409 | list_del_init(&spu->list); |
| 410 | spu_free(spu); |
| 411 | } |
| 412 | mutex_unlock(&spu_prio->active_mutex[node]); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 413 | } |
Mark Nutter | a68cf98 | 2006-10-04 17:26:12 +0200 | [diff] [blame] | 414 | kfree(spu_prio); |
Arnd Bergmann | 8b3d666 | 2005-11-15 15:53:52 -0500 | [diff] [blame] | 415 | } |