blob: d7258a103439d8316b527d64569955339489fe4c [file] [log] [blame]
Ralf Baechleb99fbc12012-09-06 11:29:53 +02001#include <linux/compiler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07002#include <linux/module.h>
3#include <linux/highmem.h>
Yoichi Yuasa52ab3202010-02-20 21:23:22 +09004#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +01005#include <linux/smp.h>
Ralf Baechlebb86bf22009-04-25 11:25:34 +02006#include <asm/fixmap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <asm/tlbflush.h>
8
Ralf Baechlebb86bf22009-04-25 11:25:34 +02009static pte_t *kmap_pte;
10
11unsigned long highstart_pfn, highend_pfn;
12
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070013void *kmap(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -070014{
15 void *addr;
16
17 might_sleep();
18 if (!PageHighMem(page))
19 return page_address(page);
20 addr = kmap_high(page);
21 flush_tlb_one((unsigned long)addr);
22
23 return addr;
24}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070025EXPORT_SYMBOL(kmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070027void kunmap(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -070028{
Ralf Baechleb72b7092009-03-30 14:49:44 +020029 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 if (!PageHighMem(page))
31 return;
32 kunmap_high(page);
33}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070034EXPORT_SYMBOL(kunmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
36/*
37 * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because
38 * no global lock is needed and because the kmap code must perform a global TLB
39 * invalidation when the kmap pool wraps.
40 *
41 * However when holding an atomic kmap is is not legal to sleep, so atomic
42 * kmaps are appropriate for short, tight code paths only.
43 */
44
Cong Wanga24401b2011-11-26 10:53:39 +080045void *kmap_atomic(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -070046{
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 unsigned long vaddr;
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070048 int idx, type;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +020050 preempt_disable();
Peter Zijlstraa8663742006-12-06 20:32:20 -080051 pagefault_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -070052 if (!PageHighMem(page))
53 return page_address(page);
54
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070055 type = kmap_atomic_idx_push();
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 idx = type + KM_TYPE_NR*smp_processor_id();
57 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
58#ifdef CONFIG_DEBUG_HIGHMEM
Ralf Baechleb72b7092009-03-30 14:49:44 +020059 BUG_ON(!pte_none(*(kmap_pte - idx)));
Linus Torvalds1da177e2005-04-16 15:20:36 -070060#endif
Ralf Baechlebb86bf22009-04-25 11:25:34 +020061 set_pte(kmap_pte-idx, mk_pte(page, PAGE_KERNEL));
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 local_flush_tlb_one((unsigned long)vaddr);
63
64 return (void*) vaddr;
65}
Cong Wanga24401b2011-11-26 10:53:39 +080066EXPORT_SYMBOL(kmap_atomic);
Linus Torvalds1da177e2005-04-16 15:20:36 -070067
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070068void __kunmap_atomic(void *kvaddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069{
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
Ralf Baechleb99fbc12012-09-06 11:29:53 +020071 int type __maybe_unused;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
73 if (vaddr < FIXADDR_START) { // FIXME
Peter Zijlstraa8663742006-12-06 20:32:20 -080074 pagefault_enable();
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +020075 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 return;
77 }
78
Peter Zijlstra20273942010-10-27 15:32:58 -070079 type = kmap_atomic_idx();
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070080#ifdef CONFIG_DEBUG_HIGHMEM
81 {
82 int idx = type + KM_TYPE_NR * smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070084 BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
85
86 /*
87 * force other mappings to Oops if they'll try to access
88 * this pte without first remap it
89 */
90 pte_clear(&init_mm, vaddr, kmap_pte-idx);
91 local_flush_tlb_one(vaddr);
92 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070093#endif
Peter Zijlstra20273942010-10-27 15:32:58 -070094 kmap_atomic_idx_pop();
Peter Zijlstraa8663742006-12-06 20:32:20 -080095 pagefault_enable();
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +020096 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -070097}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070098EXPORT_SYMBOL(__kunmap_atomic);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
Ralf Baechle60080262005-07-11 20:45:51 +0000100/*
101 * This is the same as kmap_atomic() but can map memory that doesn't
102 * have a struct page associated with it.
103 */
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700104void *kmap_atomic_pfn(unsigned long pfn)
Ralf Baechle60080262005-07-11 20:45:51 +0000105{
Ralf Baechle60080262005-07-11 20:45:51 +0000106 unsigned long vaddr;
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700107 int idx, type;
Ralf Baechle60080262005-07-11 20:45:51 +0000108
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +0200109 preempt_disable();
Peter Zijlstraa8663742006-12-06 20:32:20 -0800110 pagefault_disable();
Ralf Baechle60080262005-07-11 20:45:51 +0000111
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700112 type = kmap_atomic_idx_push();
Ralf Baechle60080262005-07-11 20:45:51 +0000113 idx = type + KM_TYPE_NR*smp_processor_id();
114 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
Ralf Baechlebb86bf22009-04-25 11:25:34 +0200115 set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL));
Ralf Baechle60080262005-07-11 20:45:51 +0000116 flush_tlb_one(vaddr);
117
118 return (void*) vaddr;
119}
120
Ralf Baechlebb86bf22009-04-25 11:25:34 +0200121void __init kmap_init(void)
122{
123 unsigned long kmap_vstart;
124
125 /* cache the first kmap pte */
126 kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
127 kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
128}