blob: 38657cdaf54d32b247c89e105611f0b96a267c75 [file] [log] [blame]
Jack Steiner14258642008-07-29 22:33:57 -07001/*
2 * SN Platform GRU Driver
3 *
4 * FAULT HANDLER FOR GRU DETECTED TLB MISSES
5 *
6 * This file contains code that handles TLB misses within the GRU.
7 * These misses are reported either via interrupts or user polling of
8 * the user CB.
9 *
10 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
26
27#include <linux/kernel.h>
28#include <linux/errno.h>
29#include <linux/spinlock.h>
30#include <linux/mm.h>
31#include <linux/hugetlb.h>
32#include <linux/device.h>
33#include <linux/io.h>
34#include <linux/uaccess.h>
Jack Steinerbb04aa72009-04-02 16:59:07 -070035#include <linux/security.h>
Jack Steiner14258642008-07-29 22:33:57 -070036#include <asm/pgtable.h>
37#include "gru.h"
38#include "grutables.h"
39#include "grulib.h"
40#include "gru_instructions.h"
41#include <asm/uv/uv_hub.h>
42
Jack Steiner9c13cb32009-12-15 16:48:14 -080043/* Return codes for vtop functions */
44#define VTOP_SUCCESS 0
45#define VTOP_INVALID -1
46#define VTOP_RETRY -2
47
48
Jack Steiner14258642008-07-29 22:33:57 -070049/*
50 * Test if a physical address is a valid GRU GSEG address
51 */
52static inline int is_gru_paddr(unsigned long paddr)
53{
54 return paddr >= gru_start_paddr && paddr < gru_end_paddr;
55}
56
57/*
58 * Find the vma of a GRU segment. Caller must hold mmap_sem.
59 */
60struct vm_area_struct *gru_find_vma(unsigned long vaddr)
61{
62 struct vm_area_struct *vma;
63
64 vma = find_vma(current->mm, vaddr);
65 if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops)
66 return vma;
67 return NULL;
68}
69
70/*
71 * Find and lock the gts that contains the specified user vaddr.
72 *
73 * Returns:
74 * - *gts with the mmap_sem locked for read and the GTS locked.
75 * - NULL if vaddr invalid OR is not a valid GSEG vaddr.
76 */
77
78static struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr)
79{
80 struct mm_struct *mm = current->mm;
81 struct vm_area_struct *vma;
82 struct gru_thread_state *gts = NULL;
83
84 down_read(&mm->mmap_sem);
85 vma = gru_find_vma(vaddr);
86 if (vma)
87 gts = gru_find_thread_state(vma, TSID(vaddr, vma));
88 if (gts)
89 mutex_lock(&gts->ts_ctxlock);
90 else
91 up_read(&mm->mmap_sem);
92 return gts;
93}
94
95static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr)
96{
97 struct mm_struct *mm = current->mm;
98 struct vm_area_struct *vma;
Jack Steinere0060432009-12-15 16:48:10 -080099 struct gru_thread_state *gts = ERR_PTR(-EINVAL);
Jack Steiner14258642008-07-29 22:33:57 -0700100
101 down_write(&mm->mmap_sem);
102 vma = gru_find_vma(vaddr);
Jack Steinere0060432009-12-15 16:48:10 -0800103 if (!vma)
104 goto err;
Jack Steiner14258642008-07-29 22:33:57 -0700105
Jack Steinere0060432009-12-15 16:48:10 -0800106 gts = gru_alloc_thread_state(vma, TSID(vaddr, vma));
107 if (IS_ERR(gts))
108 goto err;
109 mutex_lock(&gts->ts_ctxlock);
110 downgrade_write(&mm->mmap_sem);
111 return gts;
112
113err:
114 up_write(&mm->mmap_sem);
Jack Steiner14258642008-07-29 22:33:57 -0700115 return gts;
116}
117
118/*
119 * Unlock a GTS that was previously locked with gru_find_lock_gts().
120 */
121static void gru_unlock_gts(struct gru_thread_state *gts)
122{
123 mutex_unlock(&gts->ts_ctxlock);
124 up_read(&current->mm->mmap_sem);
125}
126
127/*
128 * Set a CB.istatus to active using a user virtual address. This must be done
129 * just prior to a TFH RESTART. The new cb.istatus is an in-cache status ONLY.
130 * If the line is evicted, the status may be lost. The in-cache update
131 * is necessary to prevent the user from seeing a stale cb.istatus that will
132 * change as soon as the TFH restart is complete. Races may cause an
133 * occasional failure to clear the cb.istatus, but that is ok.
Jack Steiner14258642008-07-29 22:33:57 -0700134 */
Jack Steinerb61fc692009-12-15 16:48:03 -0800135static void gru_cb_set_istatus_active(struct gru_instruction_bits *cbk)
Jack Steiner14258642008-07-29 22:33:57 -0700136{
Jack Steinerb61fc692009-12-15 16:48:03 -0800137 if (cbk) {
138 cbk->istatus = CBS_ACTIVE;
Jack Steiner14258642008-07-29 22:33:57 -0700139 }
140}
141
142/*
Jack Steiner14258642008-07-29 22:33:57 -0700143 * Read & clear a TFM
144 *
145 * The GRU has an array of fault maps. A map is private to a cpu
146 * Only one cpu will be accessing a cpu's fault map.
147 *
148 * This function scans the cpu-private fault map & clears all bits that
149 * are set. The function returns a bitmap that indicates the bits that
150 * were cleared. Note that sense the maps may be updated asynchronously by
151 * the GRU, atomic operations must be used to clear bits.
152 */
153static void get_clear_fault_map(struct gru_state *gru,
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700154 struct gru_tlb_fault_map *imap,
155 struct gru_tlb_fault_map *dmap)
Jack Steiner14258642008-07-29 22:33:57 -0700156{
157 unsigned long i, k;
158 struct gru_tlb_fault_map *tfm;
159
160 tfm = get_tfm_for_cpu(gru, gru_cpu_fault_map_id());
161 prefetchw(tfm); /* Helps on hardware, required for emulator */
162 for (i = 0; i < BITS_TO_LONGS(GRU_NUM_CBE); i++) {
163 k = tfm->fault_bits[i];
164 if (k)
165 k = xchg(&tfm->fault_bits[i], 0UL);
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700166 imap->fault_bits[i] = k;
167 k = tfm->done_bits[i];
168 if (k)
169 k = xchg(&tfm->done_bits[i], 0UL);
170 dmap->fault_bits[i] = k;
Jack Steiner14258642008-07-29 22:33:57 -0700171 }
172
173 /*
174 * Not functionally required but helps performance. (Required
175 * on emulator)
176 */
177 gru_flush_cache(tfm);
178}
179
180/*
181 * Atomic (interrupt context) & non-atomic (user context) functions to
182 * convert a vaddr into a physical address. The size of the page
183 * is returned in pageshift.
184 * returns:
185 * 0 - successful
186 * < 0 - error code
187 * 1 - (atomic only) try again in non-atomic context
188 */
189static int non_atomic_pte_lookup(struct vm_area_struct *vma,
190 unsigned long vaddr, int write,
191 unsigned long *paddr, int *pageshift)
192{
193 struct page *page;
194
Jack Steiner74ccd092009-12-15 16:48:17 -0800195#ifdef CONFIG_HUGETLB_PAGE
196 *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT;
197#else
Jack Steiner14258642008-07-29 22:33:57 -0700198 *pageshift = PAGE_SHIFT;
Jack Steiner74ccd092009-12-15 16:48:17 -0800199#endif
Jack Steiner14258642008-07-29 22:33:57 -0700200 if (get_user_pages
201 (current, current->mm, vaddr, 1, write, 0, &page, NULL) <= 0)
202 return -EFAULT;
203 *paddr = page_to_phys(page);
204 put_page(page);
205 return 0;
206}
207
208/*
Jack Steiner14258642008-07-29 22:33:57 -0700209 * atomic_pte_lookup
210 *
211 * Convert a user virtual address to a physical address
212 * Only supports Intel large pages (2MB only) on x86_64.
213 * ZZZ - hugepage support is incomplete
Jack Steiner923f7f62008-10-15 22:05:13 -0700214 *
215 * NOTE: mmap_sem is already held on entry to this function. This
216 * guarantees existence of the page tables.
Jack Steiner14258642008-07-29 22:33:57 -0700217 */
218static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr,
219 int write, unsigned long *paddr, int *pageshift)
220{
221 pgd_t *pgdp;
222 pmd_t *pmdp;
223 pud_t *pudp;
224 pte_t pte;
225
Jack Steiner14258642008-07-29 22:33:57 -0700226 pgdp = pgd_offset(vma->vm_mm, vaddr);
227 if (unlikely(pgd_none(*pgdp)))
228 goto err;
229
230 pudp = pud_offset(pgdp, vaddr);
231 if (unlikely(pud_none(*pudp)))
232 goto err;
233
234 pmdp = pmd_offset(pudp, vaddr);
235 if (unlikely(pmd_none(*pmdp)))
236 goto err;
237#ifdef CONFIG_X86_64
238 if (unlikely(pmd_large(*pmdp)))
239 pte = *(pte_t *) pmdp;
240 else
241#endif
242 pte = *pte_offset_kernel(pmdp, vaddr);
243
Jack Steiner14258642008-07-29 22:33:57 -0700244 if (unlikely(!pte_present(pte) ||
245 (write && (!pte_write(pte) || !pte_dirty(pte)))))
246 return 1;
247
248 *paddr = pte_pfn(pte) << PAGE_SHIFT;
Jack Steiner023a4072008-12-09 10:51:32 -0600249#ifdef CONFIG_HUGETLB_PAGE
Jack Steiner14258642008-07-29 22:33:57 -0700250 *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT;
Jack Steiner023a4072008-12-09 10:51:32 -0600251#else
252 *pageshift = PAGE_SHIFT;
253#endif
Jack Steiner14258642008-07-29 22:33:57 -0700254 return 0;
255
256err:
Jack Steiner14258642008-07-29 22:33:57 -0700257 return 1;
258}
259
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700260static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
261 int write, int atomic, unsigned long *gpa, int *pageshift)
262{
263 struct mm_struct *mm = gts->ts_mm;
264 struct vm_area_struct *vma;
265 unsigned long paddr;
266 int ret, ps;
267
268 vma = find_vma(mm, vaddr);
269 if (!vma)
270 goto inval;
271
272 /*
273 * Atomic lookup is faster & usually works even if called in non-atomic
274 * context.
275 */
276 rmb(); /* Must/check ms_range_active before loading PTEs */
277 ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps);
278 if (ret) {
279 if (atomic)
280 goto upm;
281 if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps))
282 goto inval;
283 }
284 if (is_gru_paddr(paddr))
285 goto inval;
286 paddr = paddr & ~((1UL << ps) - 1);
287 *gpa = uv_soc_phys_ram_to_gpa(paddr);
288 *pageshift = ps;
Jack Steiner9c13cb32009-12-15 16:48:14 -0800289 return VTOP_SUCCESS;
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700290
291inval:
Jack Steiner9c13cb32009-12-15 16:48:14 -0800292 return VTOP_INVALID;
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700293upm:
Jack Steiner9c13cb32009-12-15 16:48:14 -0800294 return VTOP_RETRY;
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700295}
296
297
Jack Steiner14258642008-07-29 22:33:57 -0700298/*
Jack Steinerc5502222009-12-15 16:48:13 -0800299 * Flush a CBE from cache. The CBE is clean in the cache. Dirty the
300 * CBE cacheline so that the line will be written back to home agent.
301 * Otherwise the line may be silently dropped. This has no impact
302 * except on performance.
303 */
304static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe)
305{
306 if (unlikely(cbe)) {
307 cbe->cbrexecstatus = 0; /* make CL dirty */
308 gru_flush_cache(cbe);
309 }
310}
311
312/*
313 * Preload the TLB with entries that may be required. Currently, preloading
314 * is implemented only for BCOPY. Preload <tlb_preload_count> pages OR to
315 * the end of the bcopy tranfer, whichever is smaller.
316 */
317static void gru_preload_tlb(struct gru_state *gru,
318 struct gru_thread_state *gts, int atomic,
319 unsigned long fault_vaddr, int asid, int write,
320 unsigned char tlb_preload_count,
321 struct gru_tlb_fault_handle *tfh,
322 struct gru_control_block_extended *cbe)
323{
324 unsigned long vaddr = 0, gpa;
325 int ret, pageshift;
326
327 if (cbe->opccpy != OP_BCOPY)
328 return;
329
330 if (fault_vaddr == cbe->cbe_baddr0)
331 vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1;
332 else if (fault_vaddr == cbe->cbe_baddr1)
333 vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1;
334
335 fault_vaddr &= PAGE_MASK;
336 vaddr &= PAGE_MASK;
337 vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE);
338
339 while (vaddr > fault_vaddr) {
340 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
341 if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write,
342 GRU_PAGESIZE(pageshift)))
343 return;
344 gru_dbg(grudev,
345 "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, rw %d, ps %d, gpa 0x%lx\n",
346 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh,
347 vaddr, asid, write, pageshift, gpa);
348 vaddr -= PAGE_SIZE;
349 STAT(tlb_preload_page);
350 }
351}
352
353/*
Jack Steiner14258642008-07-29 22:33:57 -0700354 * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
355 * Input:
356 * cb Address of user CBR. Null if not running in user context
357 * Return:
358 * 0 = dropin, exception, or switch to UPM successful
359 * 1 = range invalidate active
360 * < 0 = error code
361 *
362 */
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800363static int gru_try_dropin(struct gru_state *gru,
364 struct gru_thread_state *gts,
Jack Steiner14258642008-07-29 22:33:57 -0700365 struct gru_tlb_fault_handle *tfh,
Jack Steinerb61fc692009-12-15 16:48:03 -0800366 struct gru_instruction_bits *cbk)
Jack Steiner14258642008-07-29 22:33:57 -0700367{
Jack Steinerc5502222009-12-15 16:48:13 -0800368 struct gru_control_block_extended *cbe = NULL;
369 unsigned char tlb_preload_count = gts->ts_tlb_preload_count;
Jack Steiner563447d2009-12-15 16:48:12 -0800370 int pageshift = 0, asid, write, ret, atomic = !cbk, indexway;
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700371 unsigned long gpa = 0, vaddr = 0;
Jack Steiner14258642008-07-29 22:33:57 -0700372
373 /*
374 * NOTE: The GRU contains magic hardware that eliminates races between
375 * TLB invalidates and TLB dropins. If an invalidate occurs
376 * in the window between reading the TFH and the subsequent TLB dropin,
377 * the dropin is ignored. This eliminates the need for additional locks.
378 */
379
380 /*
Jack Steinerc5502222009-12-15 16:48:13 -0800381 * Prefetch the CBE if doing TLB preloading
382 */
383 if (unlikely(tlb_preload_count)) {
384 cbe = gru_tfh_to_cbe(tfh);
385 prefetchw(cbe);
386 }
387
388 /*
Jack Steiner14258642008-07-29 22:33:57 -0700389 * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call.
390 * Might be a hardware race OR a stupid user. Ignore FMM because FMM
391 * is a transient state.
392 */
Jack Steiner270952a2009-06-17 16:28:27 -0700393 if (tfh->status != TFHSTATUS_EXCEPTION) {
394 gru_flush_cache(tfh);
Jack Steiner67bf04a2009-12-15 16:48:11 -0800395 sync_core();
Jack Steiner270952a2009-06-17 16:28:27 -0700396 if (tfh->status != TFHSTATUS_EXCEPTION)
397 goto failnoexception;
398 STAT(tfh_stale_on_fault);
399 }
Jack Steiner14258642008-07-29 22:33:57 -0700400 if (tfh->state == TFHSTATE_IDLE)
401 goto failidle;
Jack Steinerb61fc692009-12-15 16:48:03 -0800402 if (tfh->state == TFHSTATE_MISS_FMM && cbk)
Jack Steiner14258642008-07-29 22:33:57 -0700403 goto failfmm;
404
405 write = (tfh->cause & TFHCAUSE_TLB_MOD) != 0;
406 vaddr = tfh->missvaddr;
407 asid = tfh->missasid;
Jack Steiner563447d2009-12-15 16:48:12 -0800408 indexway = tfh->indexway;
Jack Steiner14258642008-07-29 22:33:57 -0700409 if (asid == 0)
410 goto failnoasid;
411
412 rmb(); /* TFH must be cache resident before reading ms_range_active */
413
414 /*
415 * TFH is cache resident - at least briefly. Fail the dropin
416 * if a range invalidate is active.
417 */
418 if (atomic_read(&gts->ts_gms->ms_range_active))
419 goto failactive;
420
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700421 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
Jack Steiner9c13cb32009-12-15 16:48:14 -0800422 if (ret == VTOP_INVALID)
Jack Steiner14258642008-07-29 22:33:57 -0700423 goto failinval;
Jack Steiner9c13cb32009-12-15 16:48:14 -0800424 if (ret == VTOP_RETRY)
Jack Steinerecdaf2b2009-04-02 16:59:09 -0700425 goto failupm;
Jack Steiner14258642008-07-29 22:33:57 -0700426
Jack Steiner7b8274e2009-04-02 16:59:12 -0700427 if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
428 gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);
Jack Steiner99f7c222009-12-15 16:48:06 -0800429 if (atomic || !gru_update_cch(gts)) {
Jack Steiner7b8274e2009-04-02 16:59:12 -0700430 gts->ts_force_cch_reload = 1;
431 goto failupm;
432 }
433 }
Jack Steinerc5502222009-12-15 16:48:13 -0800434
435 if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800436 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
Jack Steinerc5502222009-12-15 16:48:13 -0800437 gru_flush_cache_cbe(cbe);
438 }
439
Jack Steinerb61fc692009-12-15 16:48:03 -0800440 gru_cb_set_istatus_active(cbk);
Jack Steiner5958ab82009-12-15 16:48:18 -0800441 gts->ustats.tlbdropin++;
Jack Steiner14258642008-07-29 22:33:57 -0700442 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
443 GRU_PAGESIZE(pageshift));
Jack Steiner14258642008-07-29 22:33:57 -0700444 gru_dbg(grudev,
Jack Steiner563447d2009-12-15 16:48:12 -0800445 "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
446 " rw %d, ps %d, gpa 0x%lx\n",
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800447 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid,
Jack Steiner563447d2009-12-15 16:48:12 -0800448 indexway, write, pageshift, gpa);
449 STAT(tlb_dropin);
Jack Steiner14258642008-07-29 22:33:57 -0700450 return 0;
451
452failnoasid:
453 /* No asid (delayed unload). */
454 STAT(tlb_dropin_fail_no_asid);
455 gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
Jack Steinerb61fc692009-12-15 16:48:03 -0800456 if (!cbk)
Jack Steiner14258642008-07-29 22:33:57 -0700457 tfh_user_polling_mode(tfh);
458 else
459 gru_flush_cache(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800460 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700461 return -EAGAIN;
462
463failupm:
464 /* Atomic failure switch CBR to UPM */
465 tfh_user_polling_mode(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800466 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700467 STAT(tlb_dropin_fail_upm);
468 gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
469 return 1;
470
471failfmm:
472 /* FMM state on UPM call */
Jack Steinerfe5bb6b2009-04-02 16:59:04 -0700473 gru_flush_cache(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800474 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700475 STAT(tlb_dropin_fail_fmm);
476 gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state);
477 return 0;
478
Jack Steinercd1334f2009-06-17 16:28:19 -0700479failnoexception:
480 /* TFH status did not show exception pending */
481 gru_flush_cache(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800482 gru_flush_cache_cbe(cbe);
Jack Steinerb61fc692009-12-15 16:48:03 -0800483 if (cbk)
484 gru_flush_cache(cbk);
Jack Steinercd1334f2009-06-17 16:28:19 -0700485 STAT(tlb_dropin_fail_no_exception);
Jack Steinerb61fc692009-12-15 16:48:03 -0800486 gru_dbg(grudev, "FAILED non-exception tfh: 0x%p, status %d, state %d\n",
487 tfh, tfh->status, tfh->state);
Jack Steinercd1334f2009-06-17 16:28:19 -0700488 return 0;
489
Jack Steiner14258642008-07-29 22:33:57 -0700490failidle:
Jack Steinercd1334f2009-06-17 16:28:19 -0700491 /* TFH state was idle - no miss pending */
Jack Steiner14258642008-07-29 22:33:57 -0700492 gru_flush_cache(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800493 gru_flush_cache_cbe(cbe);
Jack Steinerb61fc692009-12-15 16:48:03 -0800494 if (cbk)
495 gru_flush_cache(cbk);
Jack Steiner14258642008-07-29 22:33:57 -0700496 STAT(tlb_dropin_fail_idle);
497 gru_dbg(grudev, "FAILED idle tfh: 0x%p, state %d\n", tfh, tfh->state);
498 return 0;
499
500failinval:
501 /* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
502 tfh_exception(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800503 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700504 STAT(tlb_dropin_fail_invalid);
505 gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
506 return -EFAULT;
507
508failactive:
509 /* Range invalidate active. Switch to UPM iff atomic */
Jack Steinerb61fc692009-12-15 16:48:03 -0800510 if (!cbk)
Jack Steiner14258642008-07-29 22:33:57 -0700511 tfh_user_polling_mode(tfh);
512 else
513 gru_flush_cache(tfh);
Jack Steinerc5502222009-12-15 16:48:13 -0800514 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700515 STAT(tlb_dropin_fail_range_active);
516 gru_dbg(grudev, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n",
517 tfh, vaddr);
518 return 1;
519}
520
521/*
522 * Process an external interrupt from the GRU. This interrupt is
523 * caused by a TLB miss.
524 * Note that this is the interrupt handler that is registered with linux
525 * interrupt handlers.
526 */
Jack Steiner4107e1d2009-12-15 16:48:11 -0800527static irqreturn_t gru_intr(int chiplet, int blade)
Jack Steiner14258642008-07-29 22:33:57 -0700528{
529 struct gru_state *gru;
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700530 struct gru_tlb_fault_map imap, dmap;
Jack Steiner14258642008-07-29 22:33:57 -0700531 struct gru_thread_state *gts;
532 struct gru_tlb_fault_handle *tfh = NULL;
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800533 struct completion *cmp;
Jack Steiner14258642008-07-29 22:33:57 -0700534 int cbrnum, ctxnum;
535
536 STAT(intr);
537
Jack Steiner4107e1d2009-12-15 16:48:11 -0800538 gru = &gru_base[blade]->bs_grus[chiplet];
Jack Steiner14258642008-07-29 22:33:57 -0700539 if (!gru) {
Jack Steiner4107e1d2009-12-15 16:48:11 -0800540 dev_err(grudev, "GRU: invalid interrupt: cpu %d, chiplet %d\n",
541 raw_smp_processor_id(), chiplet);
Jack Steiner14258642008-07-29 22:33:57 -0700542 return IRQ_NONE;
543 }
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700544 get_clear_fault_map(gru, &imap, &dmap);
Jack Steiner4107e1d2009-12-15 16:48:11 -0800545 gru_dbg(grudev,
546 "cpu %d, chiplet %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n",
547 smp_processor_id(), chiplet, gru->gs_gid,
548 imap.fault_bits[0], imap.fault_bits[1],
549 dmap.fault_bits[0], dmap.fault_bits[1]);
Jack Steiner14258642008-07-29 22:33:57 -0700550
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700551 for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
Jack Steiner563447d2009-12-15 16:48:12 -0800552 STAT(intr_cbr);
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800553 cmp = gru->gs_blade->bs_async_wq;
554 if (cmp)
555 complete(cmp);
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700556 gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800557 gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
Jack Steiner4a7a17c2009-06-17 16:28:25 -0700558 }
559
560 for_each_cbr_in_tfm(cbrnum, imap.fault_bits) {
Jack Steiner563447d2009-12-15 16:48:12 -0800561 STAT(intr_tfh);
Jack Steiner14258642008-07-29 22:33:57 -0700562 tfh = get_tfh_by_index(gru, cbrnum);
563 prefetchw(tfh); /* Helps on hdw, required for emulator */
564
565 /*
566 * When hardware sets a bit in the faultmap, it implicitly
567 * locks the GRU context so that it cannot be unloaded.
568 * The gts cannot change until a TFH start/writestart command
569 * is issued.
570 */
571 ctxnum = tfh->ctxnum;
572 gts = gru->gs_gts[ctxnum];
573
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800574 /* Spurious interrupts can cause this. Ignore. */
575 if (!gts) {
576 STAT(intr_spurious);
577 continue;
578 }
579
Jack Steiner14258642008-07-29 22:33:57 -0700580 /*
581 * This is running in interrupt context. Trylock the mmap_sem.
582 * If it fails, retry the fault in user context.
583 */
Jack Steiner5958ab82009-12-15 16:48:18 -0800584 gts->ustats.fmm_tlbmiss++;
Jack Steinercd1334f2009-06-17 16:28:19 -0700585 if (!gts->ts_force_cch_reload &&
586 down_read_trylock(&gts->ts_mm->mmap_sem)) {
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800587 gru_try_dropin(gru, gts, tfh, NULL);
Jack Steiner14258642008-07-29 22:33:57 -0700588 up_read(&gts->ts_mm->mmap_sem);
589 } else {
590 tfh_user_polling_mode(tfh);
Jack Steiner43884602009-04-02 16:59:05 -0700591 STAT(intr_mm_lock_failed);
Jack Steiner14258642008-07-29 22:33:57 -0700592 }
593 }
594 return IRQ_HANDLED;
595}
596
Jack Steiner4107e1d2009-12-15 16:48:11 -0800597irqreturn_t gru0_intr(int irq, void *dev_id)
598{
599 return gru_intr(0, uv_numa_blade_id());
600}
601
602irqreturn_t gru1_intr(int irq, void *dev_id)
603{
604 return gru_intr(1, uv_numa_blade_id());
605}
606
607irqreturn_t gru_intr_mblade(int irq, void *dev_id)
608{
609 int blade;
610
611 for_each_possible_blade(blade) {
612 if (uv_blade_nr_possible_cpus(blade))
613 continue;
614 gru_intr(0, blade);
615 gru_intr(1, blade);
616 }
617 return IRQ_HANDLED;
618}
619
Jack Steiner14258642008-07-29 22:33:57 -0700620
621static int gru_user_dropin(struct gru_thread_state *gts,
622 struct gru_tlb_fault_handle *tfh,
Jack Steinerb61fc692009-12-15 16:48:03 -0800623 void *cb)
Jack Steiner14258642008-07-29 22:33:57 -0700624{
625 struct gru_mm_struct *gms = gts->ts_gms;
626 int ret;
627
Jack Steiner5958ab82009-12-15 16:48:18 -0800628 gts->ustats.upm_tlbmiss++;
Jack Steiner14258642008-07-29 22:33:57 -0700629 while (1) {
630 wait_event(gms->ms_wait_queue,
631 atomic_read(&gms->ms_range_active) == 0);
632 prefetchw(tfh); /* Helps on hdw, required for emulator */
Jack Steiner2ce4d4c2009-12-15 16:48:18 -0800633 ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb);
Jack Steiner14258642008-07-29 22:33:57 -0700634 if (ret <= 0)
635 return ret;
636 STAT(call_os_wait_queue);
637 }
638}
639
640/*
641 * This interface is called as a result of a user detecting a "call OS" bit
642 * in a user CB. Normally means that a TLB fault has occurred.
643 * cb - user virtual address of the CB
644 */
645int gru_handle_user_call_os(unsigned long cb)
646{
647 struct gru_tlb_fault_handle *tfh;
648 struct gru_thread_state *gts;
Jack Steinerb61fc692009-12-15 16:48:03 -0800649 void *cbk;
Jack Steiner14258642008-07-29 22:33:57 -0700650 int ucbnum, cbrnum, ret = -EINVAL;
651
652 STAT(call_os);
Jack Steiner14258642008-07-29 22:33:57 -0700653
654 /* sanity check the cb pointer */
655 ucbnum = get_cb_number((void *)cb);
656 if ((cb & (GRU_HANDLE_STRIDE - 1)) || ucbnum >= GRU_NUM_CB)
657 return -EINVAL;
Jack Steiner14258642008-07-29 22:33:57 -0700658
659 gts = gru_find_lock_gts(cb);
660 if (!gts)
661 return -EINVAL;
Jack Steiner563447d2009-12-15 16:48:12 -0800662 gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
Jack Steiner14258642008-07-29 22:33:57 -0700663
Jack Steinerfe5bb6b2009-04-02 16:59:04 -0700664 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
Jack Steiner14258642008-07-29 22:33:57 -0700665 goto exit;
Jack Steiner14258642008-07-29 22:33:57 -0700666
Jack Steiner55484c42009-12-15 16:48:05 -0800667 gru_check_context_placement(gts);
Jack Steinerfe5bb6b2009-04-02 16:59:04 -0700668
Jack Steiner7b8274e2009-04-02 16:59:12 -0700669 /*
670 * CCH may contain stale data if ts_force_cch_reload is set.
671 */
672 if (gts->ts_gru && gts->ts_force_cch_reload) {
Jack Steiner7b8274e2009-04-02 16:59:12 -0700673 gts->ts_force_cch_reload = 0;
Jack Steiner99f7c222009-12-15 16:48:06 -0800674 gru_update_cch(gts);
Jack Steiner7b8274e2009-04-02 16:59:12 -0700675 }
676
Jack Steiner14258642008-07-29 22:33:57 -0700677 ret = -EAGAIN;
678 cbrnum = thread_cbr_number(gts, ucbnum);
Jack Steiner99f7c222009-12-15 16:48:06 -0800679 if (gts->ts_gru) {
Jack Steiner14258642008-07-29 22:33:57 -0700680 tfh = get_tfh_by_index(gts->ts_gru, cbrnum);
Jack Steinerb61fc692009-12-15 16:48:03 -0800681 cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr,
682 gts->ts_ctxnum, ucbnum);
683 ret = gru_user_dropin(gts, tfh, cbk);
Jack Steiner14258642008-07-29 22:33:57 -0700684 }
685exit:
686 gru_unlock_gts(gts);
687 return ret;
688}
689
690/*
691 * Fetch the exception detail information for a CB that terminated with
692 * an exception.
693 */
694int gru_get_exception_detail(unsigned long arg)
695{
696 struct control_block_extended_exc_detail excdet;
697 struct gru_control_block_extended *cbe;
698 struct gru_thread_state *gts;
699 int ucbnum, cbrnum, ret;
700
701 STAT(user_exception);
702 if (copy_from_user(&excdet, (void __user *)arg, sizeof(excdet)))
703 return -EFAULT;
704
Jack Steiner14258642008-07-29 22:33:57 -0700705 gts = gru_find_lock_gts(excdet.cb);
706 if (!gts)
707 return -EINVAL;
708
Jack Steiner563447d2009-12-15 16:48:12 -0800709 gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
Jack Steinerfe5bb6b2009-04-02 16:59:04 -0700710 ucbnum = get_cb_number((void *)excdet.cb);
711 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
712 ret = -EINVAL;
713 } else if (gts->ts_gru) {
Jack Steiner14258642008-07-29 22:33:57 -0700714 cbrnum = thread_cbr_number(gts, ucbnum);
715 cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
Jack Steiner1a2c09e2009-06-17 16:28:28 -0700716 gru_flush_cache(cbe); /* CBE not coherent */
Jack Steiner67bf04a2009-12-15 16:48:11 -0800717 sync_core(); /* make sure we are have current data */
Jack Steiner14258642008-07-29 22:33:57 -0700718 excdet.opc = cbe->opccpy;
719 excdet.exopc = cbe->exopccpy;
720 excdet.ecause = cbe->ecause;
721 excdet.exceptdet0 = cbe->idef1upd;
722 excdet.exceptdet1 = cbe->idef3upd;
Jack Steinercd1334f2009-06-17 16:28:19 -0700723 excdet.cbrstate = cbe->cbrstate;
724 excdet.cbrexecstatus = cbe->cbrexecstatus;
Jack Steinerc5502222009-12-15 16:48:13 -0800725 gru_flush_cache_cbe(cbe);
Jack Steiner14258642008-07-29 22:33:57 -0700726 ret = 0;
727 } else {
728 ret = -EAGAIN;
729 }
730 gru_unlock_gts(gts);
731
Jack Steinercd1334f2009-06-17 16:28:19 -0700732 gru_dbg(grudev,
733 "cb 0x%lx, op %d, exopc %d, cbrstate %d, cbrexecstatus 0x%x, ecause 0x%x, "
734 "exdet0 0x%lx, exdet1 0x%x\n",
735 excdet.cb, excdet.opc, excdet.exopc, excdet.cbrstate, excdet.cbrexecstatus,
736 excdet.ecause, excdet.exceptdet0, excdet.exceptdet1);
Jack Steiner14258642008-07-29 22:33:57 -0700737 if (!ret && copy_to_user((void __user *)arg, &excdet, sizeof(excdet)))
738 ret = -EFAULT;
739 return ret;
740}
741
742/*
743 * User request to unload a context. Content is saved for possible reload.
744 */
Jack Steinerbb04aa72009-04-02 16:59:07 -0700745static int gru_unload_all_contexts(void)
746{
747 struct gru_thread_state *gts;
748 struct gru_state *gru;
Jack Steinere1c32192009-04-02 16:59:10 -0700749 int gid, ctxnum;
Jack Steinerbb04aa72009-04-02 16:59:07 -0700750
751 if (!capable(CAP_SYS_ADMIN))
752 return -EPERM;
Jack Steinere1c32192009-04-02 16:59:10 -0700753 foreach_gid(gid) {
Jack Steinerbb04aa72009-04-02 16:59:07 -0700754 gru = GID_TO_GRU(gid);
755 spin_lock(&gru->gs_lock);
756 for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
757 gts = gru->gs_gts[ctxnum];
758 if (gts && mutex_trylock(&gts->ts_ctxlock)) {
759 spin_unlock(&gru->gs_lock);
760 gru_unload_context(gts, 1);
Jack Steinerd57c82b2009-06-17 16:28:20 -0700761 mutex_unlock(&gts->ts_ctxlock);
Jack Steinerbb04aa72009-04-02 16:59:07 -0700762 spin_lock(&gru->gs_lock);
763 }
764 }
765 spin_unlock(&gru->gs_lock);
766 }
767 return 0;
768}
769
Jack Steiner14258642008-07-29 22:33:57 -0700770int gru_user_unload_context(unsigned long arg)
771{
772 struct gru_thread_state *gts;
773 struct gru_unload_context_req req;
774
775 STAT(user_unload_context);
776 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
777 return -EFAULT;
778
779 gru_dbg(grudev, "gseg 0x%lx\n", req.gseg);
780
Jack Steinerbb04aa72009-04-02 16:59:07 -0700781 if (!req.gseg)
782 return gru_unload_all_contexts();
783
Jack Steiner14258642008-07-29 22:33:57 -0700784 gts = gru_find_lock_gts(req.gseg);
785 if (!gts)
786 return -EINVAL;
787
788 if (gts->ts_gru)
789 gru_unload_context(gts, 1);
790 gru_unlock_gts(gts);
791
792 return 0;
793}
794
795/*
796 * User request to flush a range of virtual addresses from the GRU TLB
797 * (Mainly for testing).
798 */
799int gru_user_flush_tlb(unsigned long arg)
800{
801 struct gru_thread_state *gts;
802 struct gru_flush_tlb_req req;
Jack Steiner1926ee82009-06-17 16:28:33 -0700803 struct gru_mm_struct *gms;
Jack Steiner14258642008-07-29 22:33:57 -0700804
805 STAT(user_flush_tlb);
806 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
807 return -EFAULT;
808
809 gru_dbg(grudev, "gseg 0x%lx, vaddr 0x%lx, len 0x%lx\n", req.gseg,
810 req.vaddr, req.len);
811
812 gts = gru_find_lock_gts(req.gseg);
813 if (!gts)
814 return -EINVAL;
815
Jack Steiner1926ee82009-06-17 16:28:33 -0700816 gms = gts->ts_gms;
Jack Steiner14258642008-07-29 22:33:57 -0700817 gru_unlock_gts(gts);
Jack Steiner1926ee82009-06-17 16:28:33 -0700818 gru_flush_tlb_range(gms, req.vaddr, req.len);
Jack Steiner14258642008-07-29 22:33:57 -0700819
820 return 0;
821}
822
823/*
Jack Steiner7e796a72009-06-17 16:28:30 -0700824 * Fetch GSEG statisticss
825 */
826long gru_get_gseg_statistics(unsigned long arg)
827{
828 struct gru_thread_state *gts;
829 struct gru_get_gseg_statistics_req req;
830
831 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
832 return -EFAULT;
833
Jack Steiner091f1a12009-12-15 16:48:02 -0800834 /*
835 * The library creates arrays of contexts for threaded programs.
836 * If no gts exists in the array, the context has never been used & all
837 * statistics are implicitly 0.
838 */
Jack Steiner7e796a72009-06-17 16:28:30 -0700839 gts = gru_find_lock_gts(req.gseg);
840 if (gts) {
841 memcpy(&req.stats, &gts->ustats, sizeof(gts->ustats));
842 gru_unlock_gts(gts);
843 } else {
844 memset(&req.stats, 0, sizeof(gts->ustats));
845 }
846
847 if (copy_to_user((void __user *)arg, &req, sizeof(req)))
848 return -EFAULT;
849
850 return 0;
851}
852
853/*
Jack Steiner14258642008-07-29 22:33:57 -0700854 * Register the current task as the user of the GSEG slice.
855 * Needed for TLB fault interrupt targeting.
856 */
Jack Steiner92b39382009-06-17 16:28:32 -0700857int gru_set_context_option(unsigned long arg)
Jack Steiner14258642008-07-29 22:33:57 -0700858{
859 struct gru_thread_state *gts;
Jack Steiner92b39382009-06-17 16:28:32 -0700860 struct gru_set_context_option_req req;
861 int ret = 0;
Jack Steiner14258642008-07-29 22:33:57 -0700862
Jack Steiner92b39382009-06-17 16:28:32 -0700863 STAT(set_context_option);
864 if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
865 return -EFAULT;
866 gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1);
867
Jack Steinerc5502222009-12-15 16:48:13 -0800868 gts = gru_find_lock_gts(req.gseg);
869 if (!gts) {
870 gts = gru_alloc_locked_gts(req.gseg);
871 if (IS_ERR(gts))
872 return PTR_ERR(gts);
873 }
Jack Steiner14258642008-07-29 22:33:57 -0700874
Jack Steiner92b39382009-06-17 16:28:32 -0700875 switch (req.op) {
Jack Steiner518e5cd2009-12-15 16:48:04 -0800876 case sco_blade_chiplet:
877 /* Select blade/chiplet for GRU context */
878 if (req.val1 < -1 || req.val1 >= GRU_MAX_BLADES || !gru_base[req.val1] ||
879 req.val0 < -1 || req.val0 >= GRU_CHIPLETS_PER_HUB) {
880 ret = -EINVAL;
881 } else {
882 gts->ts_user_blade_id = req.val1;
883 gts->ts_user_chiplet_id = req.val0;
Jack Steiner55484c42009-12-15 16:48:05 -0800884 gru_check_context_placement(gts);
Jack Steiner518e5cd2009-12-15 16:48:04 -0800885 }
886 break;
Jack Steiner92b39382009-06-17 16:28:32 -0700887 case sco_gseg_owner:
888 /* Register the current task as the GSEG owner */
889 gts->ts_tgid_owner = current->tgid;
890 break;
Jack Steinerb1b19fc2009-06-17 16:28:33 -0700891 case sco_cch_req_slice:
892 /* Set the CCH slice option */
893 gts->ts_cch_req_slice = req.val1 & 3;
894 break;
Jack Steiner92b39382009-06-17 16:28:32 -0700895 default:
896 ret = -EINVAL;
897 }
Jack Steiner14258642008-07-29 22:33:57 -0700898 gru_unlock_gts(gts);
899
Jack Steiner92b39382009-06-17 16:28:32 -0700900 return ret;
Jack Steiner14258642008-07-29 22:33:57 -0700901}