Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2014 IBM Corp. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | */ |
| 9 | |
| 10 | #include <linux/module.h> |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/bitmap.h> |
| 13 | #include <linux/sched.h> |
| 14 | #include <linux/pid.h> |
| 15 | #include <linux/fs.h> |
| 16 | #include <linux/mm.h> |
| 17 | #include <linux/debugfs.h> |
| 18 | #include <linux/slab.h> |
| 19 | #include <linux/idr.h> |
| 20 | #include <asm/cputable.h> |
| 21 | #include <asm/current.h> |
| 22 | #include <asm/copro.h> |
| 23 | |
| 24 | #include "cxl.h" |
| 25 | |
| 26 | /* |
| 27 | * Allocates space for a CXL context. |
| 28 | */ |
| 29 | struct cxl_context *cxl_context_alloc(void) |
| 30 | { |
| 31 | return kzalloc(sizeof(struct cxl_context), GFP_KERNEL); |
| 32 | } |
| 33 | |
| 34 | /* |
| 35 | * Initialises a CXL context. |
| 36 | */ |
Ian Munsie | b123429 | 2014-12-08 19:18:01 +1100 | [diff] [blame] | 37 | int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master, |
| 38 | struct address_space *mapping) |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 39 | { |
| 40 | int i; |
| 41 | |
| 42 | spin_lock_init(&ctx->sste_lock); |
| 43 | ctx->afu = afu; |
| 44 | ctx->master = master; |
| 45 | ctx->pid = NULL; /* Set in start work ioctl */ |
Ian Munsie | b123429 | 2014-12-08 19:18:01 +1100 | [diff] [blame] | 46 | mutex_init(&ctx->mapping_lock); |
| 47 | ctx->mapping = mapping; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 48 | |
| 49 | /* |
| 50 | * Allocate the segment table before we put it in the IDR so that we |
| 51 | * can always access it when dereferenced from IDR. For the same |
| 52 | * reason, the segment table is only destroyed after the context is |
| 53 | * removed from the IDR. Access to this in the IOCTL is protected by |
| 54 | * Linux filesytem symantics (can't IOCTL until open is complete). |
| 55 | */ |
| 56 | i = cxl_alloc_sst(ctx); |
| 57 | if (i) |
| 58 | return i; |
| 59 | |
| 60 | INIT_WORK(&ctx->fault_work, cxl_handle_fault); |
| 61 | |
| 62 | init_waitqueue_head(&ctx->wq); |
| 63 | spin_lock_init(&ctx->lock); |
| 64 | |
| 65 | ctx->irq_bitmap = NULL; |
| 66 | ctx->pending_irq = false; |
| 67 | ctx->pending_fault = false; |
| 68 | ctx->pending_afu_err = false; |
| 69 | |
| 70 | /* |
| 71 | * When we have to destroy all contexts in cxl_context_detach_all() we |
| 72 | * end up with afu_release_irqs() called from inside a |
| 73 | * idr_for_each_entry(). Hence we need to make sure that anything |
| 74 | * dereferenced from this IDR is ok before we allocate the IDR here. |
| 75 | * This clears out the IRQ ranges to ensure this. |
| 76 | */ |
| 77 | for (i = 0; i < CXL_IRQ_RANGES; i++) |
| 78 | ctx->irqs.range[i] = 0; |
| 79 | |
| 80 | mutex_init(&ctx->status_mutex); |
| 81 | |
| 82 | ctx->status = OPENED; |
| 83 | |
| 84 | /* |
| 85 | * Allocating IDR! We better make sure everything's setup that |
| 86 | * dereferences from it. |
| 87 | */ |
Ian Munsie | ee41d11 | 2014-12-08 19:17:55 +1100 | [diff] [blame] | 88 | mutex_lock(&afu->contexts_lock); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 89 | idr_preload(GFP_KERNEL); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 90 | i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0, |
| 91 | ctx->afu->num_procs, GFP_NOWAIT); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 92 | idr_preload_end(); |
Ian Munsie | ee41d11 | 2014-12-08 19:17:55 +1100 | [diff] [blame] | 93 | mutex_unlock(&afu->contexts_lock); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 94 | if (i < 0) |
| 95 | return i; |
| 96 | |
| 97 | ctx->pe = i; |
| 98 | ctx->elem = &ctx->afu->spa[i]; |
| 99 | ctx->pe_inserted = false; |
| 100 | return 0; |
| 101 | } |
| 102 | |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 103 | static int cxl_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
| 104 | { |
| 105 | struct cxl_context *ctx = vma->vm_file->private_data; |
| 106 | unsigned long address = (unsigned long)vmf->virtual_address; |
| 107 | u64 area, offset; |
| 108 | |
| 109 | offset = vmf->pgoff << PAGE_SHIFT; |
| 110 | |
| 111 | pr_devel("%s: pe: %i address: 0x%lx offset: 0x%llx\n", |
| 112 | __func__, ctx->pe, address, offset); |
| 113 | |
| 114 | if (ctx->afu->current_mode == CXL_MODE_DEDICATED) { |
| 115 | area = ctx->afu->psn_phys; |
| 116 | if (offset > ctx->afu->adapter->ps_size) |
| 117 | return VM_FAULT_SIGBUS; |
| 118 | } else { |
| 119 | area = ctx->psn_phys; |
| 120 | if (offset > ctx->psn_size) |
| 121 | return VM_FAULT_SIGBUS; |
| 122 | } |
| 123 | |
| 124 | mutex_lock(&ctx->status_mutex); |
| 125 | |
| 126 | if (ctx->status != STARTED) { |
| 127 | mutex_unlock(&ctx->status_mutex); |
| 128 | pr_devel("%s: Context not started, failing problem state access\n", __func__); |
| 129 | return VM_FAULT_SIGBUS; |
| 130 | } |
| 131 | |
| 132 | vm_insert_pfn(vma, address, (area + offset) >> PAGE_SHIFT); |
| 133 | |
| 134 | mutex_unlock(&ctx->status_mutex); |
| 135 | |
| 136 | return VM_FAULT_NOPAGE; |
| 137 | } |
| 138 | |
| 139 | static const struct vm_operations_struct cxl_mmap_vmops = { |
| 140 | .fault = cxl_mmap_fault, |
| 141 | }; |
| 142 | |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 143 | /* |
| 144 | * Map a per-context mmio space into the given vma. |
| 145 | */ |
| 146 | int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma) |
| 147 | { |
| 148 | u64 len = vma->vm_end - vma->vm_start; |
| 149 | len = min(len, ctx->psn_size); |
| 150 | |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 151 | if (ctx->afu->current_mode != CXL_MODE_DEDICATED) { |
| 152 | /* make sure there is a valid per process space for this AFU */ |
| 153 | if ((ctx->master && !ctx->afu->psa) || (!ctx->afu->pp_psa)) { |
| 154 | pr_devel("AFU doesn't support mmio space\n"); |
| 155 | return -EINVAL; |
| 156 | } |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 157 | |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 158 | /* Can't mmap until the AFU is enabled */ |
| 159 | if (!ctx->afu->enabled) |
| 160 | return -EBUSY; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 161 | } |
| 162 | |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 163 | pr_devel("%s: mmio physical: %llx pe: %i master:%i\n", __func__, |
| 164 | ctx->psn_phys, ctx->pe , ctx->master); |
| 165 | |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 166 | vma->vm_flags |= VM_IO | VM_PFNMAP; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 167 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 168 | vma->vm_ops = &cxl_mmap_vmops; |
| 169 | return 0; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | /* |
| 173 | * Detach a context from the hardware. This disables interrupts and doesn't |
| 174 | * return until all outstanding interrupts for this context have completed. The |
| 175 | * hardware should no longer access *ctx after this has returned. |
| 176 | */ |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 177 | int __detach_context(struct cxl_context *ctx) |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 178 | { |
| 179 | enum cxl_context_status status; |
| 180 | |
| 181 | mutex_lock(&ctx->status_mutex); |
| 182 | status = ctx->status; |
| 183 | ctx->status = CLOSED; |
| 184 | mutex_unlock(&ctx->status_mutex); |
| 185 | if (status != STARTED) |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 186 | return -EBUSY; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 187 | |
| 188 | WARN_ON(cxl_detach_process(ctx)); |
Michael Neuling | 7bb5d91a | 2015-05-27 16:07:14 +1000 | [diff] [blame] | 189 | flush_work(&ctx->fault_work); /* Only needed for dedicated process */ |
| 190 | put_pid(ctx->pid); |
| 191 | cxl_ctx_put(); |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 192 | return 0; |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 193 | } |
| 194 | |
| 195 | /* |
| 196 | * Detach the given context from the AFU. This doesn't actually |
| 197 | * free the context but it should stop the context running in hardware |
| 198 | * (ie. prevent this context from generating any further interrupts |
| 199 | * so that it can be freed). |
| 200 | */ |
| 201 | void cxl_context_detach(struct cxl_context *ctx) |
| 202 | { |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 203 | int rc; |
| 204 | |
| 205 | rc = __detach_context(ctx); |
| 206 | if (rc) |
| 207 | return; |
| 208 | |
| 209 | afu_release_irqs(ctx, ctx); |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 210 | wake_up_all(&ctx->wq); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 211 | } |
| 212 | |
| 213 | /* |
| 214 | * Detach all contexts on the given AFU. |
| 215 | */ |
| 216 | void cxl_context_detach_all(struct cxl_afu *afu) |
| 217 | { |
| 218 | struct cxl_context *ctx; |
| 219 | int tmp; |
| 220 | |
Ian Munsie | ee41d11 | 2014-12-08 19:17:55 +1100 | [diff] [blame] | 221 | mutex_lock(&afu->contexts_lock); |
| 222 | idr_for_each_entry(&afu->contexts_idr, ctx, tmp) { |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 223 | /* |
| 224 | * Anything done in here needs to be setup before the IDR is |
| 225 | * created and torn down after the IDR removed |
| 226 | */ |
Michael Neuling | eda3693 | 2015-05-27 16:07:08 +1000 | [diff] [blame] | 227 | cxl_context_detach(ctx); |
Ian Munsie | 0712dc7 | 2015-01-07 16:33:04 +1100 | [diff] [blame] | 228 | |
| 229 | /* |
| 230 | * We are force detaching - remove any active PSA mappings so |
| 231 | * userspace cannot interfere with the card if it comes back. |
| 232 | * Easiest way to exercise this is to unbind and rebind the |
| 233 | * driver via sysfs while it is in use. |
| 234 | */ |
| 235 | mutex_lock(&ctx->mapping_lock); |
| 236 | if (ctx->mapping) |
| 237 | unmap_mapping_range(ctx->mapping, 0, 0, 1); |
| 238 | mutex_unlock(&ctx->mapping_lock); |
Ian Munsie | ee41d11 | 2014-12-08 19:17:55 +1100 | [diff] [blame] | 239 | } |
| 240 | mutex_unlock(&afu->contexts_lock); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 241 | } |
| 242 | |
Ian Munsie | 8ac75b9 | 2015-05-08 22:55:18 +1000 | [diff] [blame] | 243 | static void reclaim_ctx(struct rcu_head *rcu) |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 244 | { |
Ian Munsie | 8ac75b9 | 2015-05-08 22:55:18 +1000 | [diff] [blame] | 245 | struct cxl_context *ctx = container_of(rcu, struct cxl_context, rcu); |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 246 | |
| 247 | free_page((u64)ctx->sstp); |
| 248 | ctx->sstp = NULL; |
| 249 | |
Ian Munsie | f204e0b | 2014-10-08 19:55:02 +1100 | [diff] [blame] | 250 | kfree(ctx); |
| 251 | } |
Ian Munsie | 8ac75b9 | 2015-05-08 22:55:18 +1000 | [diff] [blame] | 252 | |
| 253 | void cxl_context_free(struct cxl_context *ctx) |
| 254 | { |
| 255 | mutex_lock(&ctx->afu->contexts_lock); |
| 256 | idr_remove(&ctx->afu->contexts_idr, ctx->pe); |
| 257 | mutex_unlock(&ctx->afu->contexts_lock); |
| 258 | call_rcu(&ctx->rcu, reclaim_ctx); |
| 259 | } |