blob: 7ac30612068b674e742fb5577ed169148347f03f [file] [log] [blame]
Jes Sorensen17a3b052006-09-27 01:50:11 -07001/*
2 * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights
3 * reserved.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2 of the GNU General Public License
7 * as published by the Free Software Foundation.
8 */
9
10/*
11 * SN Platform Special Memory (mspec) Support
12 *
13 * This driver exports the SN special memory (mspec) facility to user
14 * processes.
15 * There are three types of memory made available thru this driver:
16 * fetchops, uncached and cached.
17 *
18 * Fetchops are atomic memory operations that are implemented in the
19 * memory controller on SGI SN hardware.
20 *
21 * Uncached are used for memory write combining feature of the ia64
22 * cpu.
23 *
24 * Cached are used for areas of memory that are used as cached addresses
25 * on our partition and used as uncached addresses from other partitions.
26 * Due to a design constraint of the SN2 Shub, you can not have processors
27 * on the same FSB perform both a cached and uncached reference to the
28 * same cache line. These special memory cached regions prevent the
29 * kernel from ever dropping in a TLB entry and therefore prevent the
30 * processor from ever speculating a cache line from this page.
31 */
32
Jes Sorensen17a3b052006-09-27 01:50:11 -070033#include <linux/types.h>
34#include <linux/kernel.h>
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/errno.h>
38#include <linux/miscdevice.h>
39#include <linux/spinlock.h>
40#include <linux/mm.h>
41#include <linux/vmalloc.h>
42#include <linux/string.h>
43#include <linux/slab.h>
44#include <linux/numa.h>
45#include <asm/page.h>
46#include <asm/system.h>
47#include <asm/pgtable.h>
48#include <asm/atomic.h>
49#include <asm/tlbflush.h>
50#include <asm/uncached.h>
51#include <asm/sn/addrs.h>
52#include <asm/sn/arch.h>
53#include <asm/sn/mspec.h>
54#include <asm/sn/sn_cpuid.h>
55#include <asm/sn/io.h>
56#include <asm/sn/bte.h>
57#include <asm/sn/shubio.h>
58
59
60#define FETCHOP_ID "SGI Fetchop,"
61#define CACHED_ID "Cached,"
62#define UNCACHED_ID "Uncached"
63#define REVISION "4.0"
64#define MSPEC_BASENAME "mspec"
65
66/*
67 * Page types allocated by the device.
68 */
69enum {
70 MSPEC_FETCHOP = 1,
71 MSPEC_CACHED,
72 MSPEC_UNCACHED
73};
74
Jes Sorensen1a4b0fc2006-11-10 12:27:49 -080075#ifdef CONFIG_SGI_SN
Jes Sorensen17a3b052006-09-27 01:50:11 -070076static int is_sn2;
Jes Sorensen1a4b0fc2006-11-10 12:27:49 -080077#else
78#define is_sn2 0
79#endif
Jes Sorensen17a3b052006-09-27 01:50:11 -070080
81/*
82 * One of these structures is allocated when an mspec region is mmaped. The
83 * structure is pointed to by the vma->vm_private_data field in the vma struct.
84 * This structure is used to record the addresses of the mspec pages.
85 */
86struct vma_data {
87 atomic_t refcnt; /* Number of vmas sharing the data. */
88 spinlock_t lock; /* Serialize access to the vma. */
89 int count; /* Number of pages allocated. */
90 int type; /* Type of pages allocated. */
91 unsigned long maddr[0]; /* Array of MSPEC addresses. */
92};
93
94/* used on shub2 to clear FOP cache in the HUB */
95static unsigned long scratch_page[MAX_NUMNODES];
96#define SH2_AMO_CACHE_ENTRIES 4
97
98static inline int
99mspec_zero_block(unsigned long addr, int len)
100{
101 int status;
102
103 if (is_sn2) {
104 if (is_shub2()) {
105 int nid;
106 void *p;
107 int i;
108
109 nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
110 p = (void *)TO_AMO(scratch_page[nid]);
111
112 for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
113 FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
114 p += FETCHOP_VAR_SIZE;
115 }
116 }
117
118 status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
119 BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
120 } else {
121 memset((char *) addr, 0, len);
122 status = 0;
123 }
124 return status;
125}
126
127/*
128 * mspec_open
129 *
130 * Called when a device mapping is created by a means other than mmap
131 * (via fork, etc.). Increments the reference count on the underlying
132 * mspec data so it is not freed prematurely.
133 */
134static void
135mspec_open(struct vm_area_struct *vma)
136{
137 struct vma_data *vdata;
138
139 vdata = vma->vm_private_data;
140 atomic_inc(&vdata->refcnt);
141}
142
143/*
144 * mspec_close
145 *
146 * Called when unmapping a device mapping. Frees all mspec pages
147 * belonging to the vma.
148 */
149static void
150mspec_close(struct vm_area_struct *vma)
151{
152 struct vma_data *vdata;
153 int i, pages, result, vdata_size;
154
155 vdata = vma->vm_private_data;
156 if (!atomic_dec_and_test(&vdata->refcnt))
157 return;
158
159 pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
160 vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
161 for (i = 0; i < pages; i++) {
162 if (vdata->maddr[i] == 0)
163 continue;
164 /*
165 * Clear the page before sticking it back
166 * into the pool.
167 */
168 result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
169 if (!result)
170 uncached_free_page(vdata->maddr[i]);
171 else
172 printk(KERN_WARNING "mspec_close(): "
173 "failed to zero page %i\n",
174 result);
175 }
176
177 if (vdata_size <= PAGE_SIZE)
178 kfree(vdata);
179 else
180 vfree(vdata);
181}
182
183
184/*
185 * mspec_nopfn
186 *
187 * Creates a mspec page and maps it to user space.
188 */
189static unsigned long
190mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
191{
192 unsigned long paddr, maddr;
193 unsigned long pfn;
194 int index;
195 struct vma_data *vdata = vma->vm_private_data;
196
197 index = (address - vma->vm_start) >> PAGE_SHIFT;
198 maddr = (volatile unsigned long) vdata->maddr[index];
199 if (maddr == 0) {
200 maddr = uncached_alloc_page(numa_node_id());
201 if (maddr == 0)
202 return NOPFN_OOM;
203
204 spin_lock(&vdata->lock);
205 if (vdata->maddr[index] == 0) {
206 vdata->count++;
207 vdata->maddr[index] = maddr;
208 } else {
209 uncached_free_page(maddr);
210 maddr = vdata->maddr[index];
211 }
212 spin_unlock(&vdata->lock);
213 }
214
215 if (vdata->type == MSPEC_FETCHOP)
216 paddr = TO_AMO(maddr);
217 else
Jes Sorensen1a4b0fc2006-11-10 12:27:49 -0800218 paddr = maddr & ~__IA64_UNCACHED_OFFSET;
Jes Sorensen17a3b052006-09-27 01:50:11 -0700219
220 pfn = paddr >> PAGE_SHIFT;
221
222 return pfn;
223}
224
225static struct vm_operations_struct mspec_vm_ops = {
226 .open = mspec_open,
227 .close = mspec_close,
228 .nopfn = mspec_nopfn
229};
230
231/*
232 * mspec_mmap
233 *
234 * Called when mmaping the device. Initializes the vma with a fault handler
235 * and private data structure necessary to allocate, track, and free the
236 * underlying pages.
237 */
238static int
239mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
240{
241 struct vma_data *vdata;
242 int pages, vdata_size;
243
244 if (vma->vm_pgoff != 0)
245 return -EINVAL;
246
247 if ((vma->vm_flags & VM_SHARED) == 0)
248 return -EINVAL;
249
250 if ((vma->vm_flags & VM_WRITE) == 0)
251 return -EPERM;
252
253 pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
254 vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
255 if (vdata_size <= PAGE_SIZE)
256 vdata = kmalloc(vdata_size, GFP_KERNEL);
257 else
258 vdata = vmalloc(vdata_size);
259 if (!vdata)
260 return -ENOMEM;
261 memset(vdata, 0, vdata_size);
262
263 vdata->type = type;
264 spin_lock_init(&vdata->lock);
265 vdata->refcnt = ATOMIC_INIT(1);
266 vma->vm_private_data = vdata;
267
268 vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
269 if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
270 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
271 vma->vm_ops = &mspec_vm_ops;
272
273 return 0;
274}
275
276static int
277fetchop_mmap(struct file *file, struct vm_area_struct *vma)
278{
279 return mspec_mmap(file, vma, MSPEC_FETCHOP);
280}
281
282static int
283cached_mmap(struct file *file, struct vm_area_struct *vma)
284{
285 return mspec_mmap(file, vma, MSPEC_CACHED);
286}
287
288static int
289uncached_mmap(struct file *file, struct vm_area_struct *vma)
290{
291 return mspec_mmap(file, vma, MSPEC_UNCACHED);
292}
293
Arjan van de Ven2b8693c2007-02-12 00:55:32 -0800294static const struct file_operations fetchop_fops = {
Jes Sorensen17a3b052006-09-27 01:50:11 -0700295 .owner = THIS_MODULE,
296 .mmap = fetchop_mmap
297};
298
299static struct miscdevice fetchop_miscdev = {
300 .minor = MISC_DYNAMIC_MINOR,
301 .name = "sgi_fetchop",
302 .fops = &fetchop_fops
303};
304
Arjan van de Ven2b8693c2007-02-12 00:55:32 -0800305static const struct file_operations cached_fops = {
Jes Sorensen17a3b052006-09-27 01:50:11 -0700306 .owner = THIS_MODULE,
307 .mmap = cached_mmap
308};
309
310static struct miscdevice cached_miscdev = {
311 .minor = MISC_DYNAMIC_MINOR,
312 .name = "mspec_cached",
313 .fops = &cached_fops
314};
315
Arjan van de Ven2b8693c2007-02-12 00:55:32 -0800316static const struct file_operations uncached_fops = {
Jes Sorensen17a3b052006-09-27 01:50:11 -0700317 .owner = THIS_MODULE,
318 .mmap = uncached_mmap
319};
320
321static struct miscdevice uncached_miscdev = {
322 .minor = MISC_DYNAMIC_MINOR,
323 .name = "mspec_uncached",
324 .fops = &uncached_fops
325};
326
327/*
328 * mspec_init
329 *
330 * Called at boot time to initialize the mspec facility.
331 */
332static int __init
333mspec_init(void)
334{
335 int ret;
336 int nid;
337
338 /*
339 * The fetchop device only works on SN2 hardware, uncached and cached
340 * memory drivers should both be valid on all ia64 hardware
341 */
Jes Sorensen1a4b0fc2006-11-10 12:27:49 -0800342#ifdef CONFIG_SGI_SN
Jes Sorensen17a3b052006-09-27 01:50:11 -0700343 if (ia64_platform_is("sn2")) {
344 is_sn2 = 1;
345 if (is_shub2()) {
346 ret = -ENOMEM;
347 for_each_online_node(nid) {
348 int actual_nid;
349 int nasid;
350 unsigned long phys;
351
352 scratch_page[nid] = uncached_alloc_page(nid);
353 if (scratch_page[nid] == 0)
354 goto free_scratch_pages;
355 phys = __pa(scratch_page[nid]);
356 nasid = get_node_number(phys);
357 actual_nid = nasid_to_cnodeid(nasid);
358 if (actual_nid != nid)
359 goto free_scratch_pages;
360 }
361 }
362
363 ret = misc_register(&fetchop_miscdev);
364 if (ret) {
365 printk(KERN_ERR
366 "%s: failed to register device %i\n",
367 FETCHOP_ID, ret);
368 goto free_scratch_pages;
369 }
370 }
Jes Sorensen1a4b0fc2006-11-10 12:27:49 -0800371#endif
Jes Sorensen17a3b052006-09-27 01:50:11 -0700372 ret = misc_register(&cached_miscdev);
373 if (ret) {
374 printk(KERN_ERR "%s: failed to register device %i\n",
375 CACHED_ID, ret);
376 if (is_sn2)
377 misc_deregister(&fetchop_miscdev);
378 goto free_scratch_pages;
379 }
380 ret = misc_register(&uncached_miscdev);
381 if (ret) {
382 printk(KERN_ERR "%s: failed to register device %i\n",
383 UNCACHED_ID, ret);
384 misc_deregister(&cached_miscdev);
385 if (is_sn2)
386 misc_deregister(&fetchop_miscdev);
387 goto free_scratch_pages;
388 }
389
390 printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
391 MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
392 CACHED_ID, UNCACHED_ID);
393
394 return 0;
395
396 free_scratch_pages:
397 for_each_node(nid) {
398 if (scratch_page[nid] != 0)
399 uncached_free_page(scratch_page[nid]);
400 }
401 return ret;
402}
403
404static void __exit
405mspec_exit(void)
406{
407 int nid;
408
409 misc_deregister(&uncached_miscdev);
410 misc_deregister(&cached_miscdev);
411 if (is_sn2) {
412 misc_deregister(&fetchop_miscdev);
413
414 for_each_node(nid) {
415 if (scratch_page[nid] != 0)
416 uncached_free_page(scratch_page[nid]);
417 }
418 }
419}
420
421module_init(mspec_init);
422module_exit(mspec_exit);
423
424MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
425MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
426MODULE_LICENSE("GPL");